diff --git a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json index e7ed0aee65..d29a18c691 100644 --- a/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json +++ b/backend/.sqlx/query-07168aaf14cb6beff0ad4274b441f7f387f5055c47f493271d26731336257384.json @@ -46,11 +46,11 @@ ] }, "nullable": [ - false, - false, - false, - false, - false, + true, + true, + true, + true, + true, true, true ] diff --git a/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json b/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json index ecaf828737..bc15928e38 100644 --- a/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json +++ b/backend/.sqlx/query-0a7132202ecf6c4c10340921644a90d9206c45d92a0423c0bc2396d0d66a0b0d.json @@ -59,7 +59,9 @@ "failure", "command", "approval", - "preprocessor" + "preprocessor", + "schedule_handler_old", + "dynamic_skip" ] } } diff --git a/backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json b/backend/.sqlx/query-76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64.json similarity index 69% rename from backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json rename to backend/.sqlx/query-76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64.json index 50d4a9595d..f0c32e5dd0 100644 --- a/backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json +++ b/backend/.sqlx/query-76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9, job_isolation = $10 WHERE worker = $5", "describe": { "columns": [], "parameters": { @@ -13,10 +13,11 @@ "Float4", "Float4", "Float4", - "Float4" + "Float4", + "Text" ] }, "nullable": [] }, - "hash": "506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773" + "hash": "76a7ad0588afcb4e9f0b876c50e203c36be56b2b2b08ca787417a9822ef56f64" } diff --git a/backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json b/backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json similarity index 80% rename from backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json rename to backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json index 9df7dced7b..644cb25123 100644 --- a/backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json +++ b/backend/.sqlx/query-771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,\n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -97,6 +97,11 @@ "ordinal": 18, "name": "wm_memory_usage", "type_info": "Int8" + }, + { + "ordinal": 19, + "name": "job_isolation", + "type_info": "Text" } ], "parameters": { @@ -126,8 +131,9 @@ true, true, true, + true, true ] }, - "hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab" + "hash": "771a858a4b7ca41b6787e61f5a4a5c9c4d48fd213852e2f997cd4b2420580d30" } diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 54e94cfb8f..99269c9851 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - false, - true + true, + false ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json b/backend/.sqlx/query-ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c.json similarity index 56% rename from backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json rename to backend/.sqlx/query-ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c.json index 3867319977..15e83648ec 100644 --- a/backend/.sqlx/query-e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2.json +++ b/backend/.sqlx/query-ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) \n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", + "query": "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory, job_isolation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (worker)\n DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", "describe": { "columns": [], "parameters": { @@ -13,10 +13,11 @@ "Varchar", "Varchar", "Int8", - "Int8" + "Int8", + "Text" ] }, "nullable": [] }, - "hash": "e5f8830450e90f678494ae2b0f86b0692a02dec4c8415fd7d364b0cc088905c2" + "hash": "ed90b9fcc57f530bab2d2d7426795bb358c91ebdd9dab50d7e3d2fdce63b947c" } diff --git a/backend/migrations/20251112200815_add_enable_unshare_pid.down.sql b/backend/migrations/20251112200815_add_enable_unshare_pid.down.sql new file mode 100644 index 0000000000..d6fbf14f23 --- /dev/null +++ b/backend/migrations/20251112200815_add_enable_unshare_pid.down.sql @@ -0,0 +1,2 @@ +-- Rollback: Remove job_isolation column from worker_ping table +ALTER TABLE worker_ping DROP COLUMN IF EXISTS job_isolation; diff --git a/backend/migrations/20251112200815_add_enable_unshare_pid.up.sql b/backend/migrations/20251112200815_add_enable_unshare_pid.up.sql new file mode 100644 index 0000000000..b7b2674ebd --- /dev/null +++ b/backend/migrations/20251112200815_add_enable_unshare_pid.up.sql @@ -0,0 +1,4 @@ +-- Add job_isolation column to worker_ping table +-- This tracks which job isolation method the worker is using: 'nsjail', 'unshare', or 'none' +-- Nullable for backwards compatibility - old workers will report NULL +ALTER TABLE worker_ping ADD COLUMN job_isolation TEXT; diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 85740d0d2f..e3d9404018 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -18802,6 +18802,8 @@ components: type: number wm_memory_usage: type: number + job_isolation: + type: string required: - worker - worker_instance diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 4e6c5ab086..1f49ec5492 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -69,6 +69,8 @@ struct WorkerPing { memory_usage: Option, #[serde(skip_serializing_if = "Option::is_none")] wm_memory_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + job_isolation: Option, } // #[derive(Serialize, Deserialize)] @@ -97,8 +99,8 @@ async fn list_worker_pings( let rows = sqlx::query_as!( WorkerPing, "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, - CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, - custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage + CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, + custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage, job_isolation FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 0b0055ed0d..2918d40b0c 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1301,6 +1301,7 @@ pub struct Ping { pub occupancy_rate_15s: Option, pub occupancy_rate_5m: Option, pub occupancy_rate_30m: Option, + pub job_isolation: Option, pub ping_type: PingType, } pub async fn update_ping_http( @@ -1348,6 +1349,7 @@ pub async fn update_ping_http( &insert_ping.version.unwrap(), insert_ping.vcpus, insert_ping.memory, + insert_ping.job_isolation, db, ) .await?; @@ -1363,6 +1365,7 @@ pub async fn update_ping_http( insert_ping.occupancy_rate_15s, insert_ping.occupancy_rate_5m, insert_ping.occupancy_rate_30m, + insert_ping.job_isolation, db, ) .await?; @@ -1476,10 +1479,11 @@ pub async fn insert_ping_query( version: &str, vcpus: Option, memory: Option, + job_isolation: Option, db: &DB, ) -> anyhow::Result<()> { sqlx::query!( - "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) + "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory, job_isolation) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (worker) DO UPDATE set ip = EXCLUDED.ip, custom_tags = EXCLUDED.custom_tags, worker_group = EXCLUDED.worker_group", worker_instance, worker_name, @@ -1489,7 +1493,8 @@ pub async fn insert_ping_query( dw, version, vcpus, - memory + memory, + job_isolation.as_deref() ) .execute(db) .await?; @@ -1506,11 +1511,12 @@ pub async fn update_worker_ping_from_job_query( occupancy_rate_15s: Option, occupancy_rate_5m: Option, occupancy_rate_30m: Option, + job_isolation: Option, db: &DB, ) -> anyhow::Result<()> { sqlx::query!( "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4, - occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5", + occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9, job_isolation = $10 WHERE worker = $5", job_id, w_id, memory_usage, @@ -1520,6 +1526,7 @@ pub async fn update_worker_ping_from_job_query( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + job_isolation, ) .execute(db) .await?; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index ac1ae4e233..9aed0e18ad 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -29,13 +29,13 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ bash_executor::BIN_BASH, common::{ - check_executor_binary_exists, get_reserved_variables, read_and_check_result, - start_child_process, transform_json, OccupancyMetrics, + build_command_with_isolation, check_executor_binary_exists, get_reserved_variables, + read_and_check_result, start_child_process, transform_json, OccupancyMetrics, }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, - PyVAlias, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, PyVAlias, TZ_ENV, }; use windmill_common::client::AuthedClient; @@ -1240,7 +1240,11 @@ fi .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let mut ansible_cmd = Command::new(ANSIBLE_PLAYBOOK_PATH.as_str()); + let ansible_args: Vec<&str> = cmd_args.iter().map(|s| s.as_ref()).collect(); + let mut ansible_cmd = build_command_with_isolation( + ANSIBLE_PLAYBOOK_PATH.as_str(), + &ansible_args, + ); ansible_cmd .current_dir(job_dir) .env_clear() @@ -1250,7 +1254,6 @@ fi .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index d147aaff9b..640f3c9fae 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -35,7 +35,7 @@ use crate::handle_child::run_future_with_polling_update_job_poller; use crate::{ common::{ - build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, + build_args_map, build_command_with_isolation, get_reserved_variables, read_file, read_file_content, start_child_process, OccupancyMetrics, }, handle_child::handle_child, @@ -200,7 +200,10 @@ exit $exit_status } else { let mut cmd_args = vec!["wrapper.sh"]; cmd_args.extend(&args); - let mut bash_cmd = Command::new(BIN_BASH.as_str()); + let mut bash_cmd = build_command_with_isolation( + BIN_BASH.as_str(), + &cmd_args.iter().map(|s| s.as_ref()).collect::>(), + ); bash_cmd .current_dir(job_dir) .env_clear() @@ -209,7 +212,6 @@ exit $exit_status .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(cmd_args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 757656b934..b8723ad815 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -17,7 +17,7 @@ use crate::common::build_envs_map; use crate::{ common::{ create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, - read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, + build_command_with_isolation, read_file_content, read_result, start_child_process, write_file_binary, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, @@ -1430,15 +1430,15 @@ try {{ } else { let cmd = if annotation.nodejs { let script_path = format!("{job_dir}/wrapper.mjs"); + let args = vec!["--preserve-symlinks", script_path.as_str()]; - let mut bun_cmd = Command::new(&*NODE_BIN_PATH); + let mut bun_cmd = build_command_with_isolation(&*NODE_BIN_PATH, &args); bun_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_bun_proc_envs) - .args(vec!["--preserve-symlinks", &script_path]) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1450,8 +1450,7 @@ try {{ } else { let script_path = format!("{job_dir}/wrapper.mjs"); - let mut bun_cmd = Command::new(&*BUN_PATH); - let args = if codebase.is_some() || has_bundle_cache { + let args: Vec<&str> = if codebase.is_some() || has_bundle_cache { vec!["run", &script_path] } else { vec![ @@ -1463,13 +1462,13 @@ try {{ &script_path, ] }; + let mut bun_cmd = build_command_with_isolation(&*BUN_PATH, &args); bun_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_bun_proc_envs) - .args(args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -1480,16 +1479,12 @@ try {{ bun_cmd }; - start_child_process( - cmd, - if annotation.nodejs { - &*NODE_BIN_PATH - } else { - &*BUN_PATH - }, - false, - ) - .await? + let executable = if annotation.nodejs { + &*NODE_BIN_PATH + } else { + &*BUN_PATH + }; + start_child_process(cmd, executable, false).await? }; let stream_notifier = StreamNotifier::new(conn, job); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 4a6edbd55d..2d995ea9bb 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -528,6 +528,7 @@ pub async fn update_worker_ping_for_failed_init_script( memory: None, memory_usage: None, wm_memory_usage: None, + job_isolation: None, ping_type: PingType::InitScript, }, ) @@ -628,6 +629,38 @@ lazy_static! { static ref DISABLE_PROCESS_GROUP: bool = std::env::var("DISABLE_PROCESS_GROUP").is_ok(); } +pub fn build_command_with_isolation( + program: &str, + args: &[&str], +) -> Command { + use tokio::process::Command; + + if *crate::ENABLE_UNSHARE_PID { + if let Some(unshare_path) = crate::UNSHARE_PATH.as_ref() { + let mut cmd = Command::new(unshare_path); + + let flags = crate::UNSHARE_ISOLATION_FLAGS.as_str(); + for flag in flags.split_whitespace() { + cmd.arg(flag); + } + + cmd.arg("--"); + cmd.arg(program); + cmd.args(args); + cmd + } else { + panic!( + "BUG: ENABLE_UNSHARE_PID is true but UNSHARE_PATH is None. \ + This should have been caught at worker startup." + ); + } + } else { + let mut cmd = Command::new(program); + cmd.args(args); + cmd + } +} + pub async fn start_child_process( cmd: Command, executable: &str, diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index 51bdf2f8e8..d5a91ab61e 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -27,12 +27,12 @@ use windmill_queue::CanceledBy; #[cfg(feature = "csharp")] use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, + get_reserved_variables, read_result, start_child_process, }, handle_child::handle_child, - CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, - NUGET_CONFIG, PATH_ENV, TZ_ENV, + CSHARP_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, DOTNET_PATH, HOME_ENV, + NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, TZ_ENV, }; use crate::common::OccupancyMetrics; @@ -588,7 +588,8 @@ pub async fn handle_csharp_job( } else { format!("{job_dir}/Main.exe") }; - let mut run_csharp = Command::new(&compiled_executable_name); + + let mut run_csharp = build_command_with_isolation(&compiled_executable_name, &[]); run_csharp .current_dir(job_dir) .env_clear() diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index cda1f78fdc..332cd142db 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,7 +7,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, @@ -406,14 +406,17 @@ try {{ args.push("-A"); } args.push(&script_path); - let mut deno_cmd = Command::new(DENO_PATH.as_str()); + + let mut deno_cmd = build_command_with_isolation( + DENO_PATH.as_str(), + &args.iter().map(|s| s.as_ref()).collect::>(), + ); deno_cmd .current_dir(job_dir) .env_clear() .envs(envs) .envs(reserved_variables) .envs(common_deno_proc_envs) - .args(args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -589,7 +592,7 @@ BigInt.prototype.toJSON = function () {{ {dates} -console.log('start\n'); +console.log('start\n'); const decoder = new TextDecoder(); for await (const chunk of Deno.stdin.readable) {{ @@ -601,7 +604,7 @@ for await (const chunk of Deno.stdin.readable) {{ break; }} try {{ - let {{ {spread} }} = JSON.parse(line) + let {{ {spread} }} = JSON.parse(line) {dates} let res: any = await main(...[ {spread} ]); console.log("wm_res[success]:" + JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value) + '\n'); diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 0409ceba95..bf3957c430 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -19,7 +19,7 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - capitalize, create_args_and_out_file, get_reserved_variables, read_result, + build_command_with_isolation, capitalize, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, @@ -371,10 +371,7 @@ func Run(req Req) (interface{{}}, error){{ #[cfg(windows)] let compiled_executable_name = format!("{}/main.exe", job_dir); - #[cfg(unix)] - let mut run_go = Command::new(&compiled_executable_name); - #[cfg(windows)] - let mut run_go = Command::new(&compiled_executable_name); + let mut run_go = build_command_with_isolation(&compiled_executable_name, &[]); run_go .current_dir(job_dir) diff --git a/backend/windmill-worker/src/java_executor.rs b/backend/windmill-worker/src/java_executor.rs index 92ef69da16..b553b9eff1 100644 --- a/backend/windmill-worker/src/java_executor.rs +++ b/backend/windmill-worker/src/java_executor.rs @@ -21,13 +21,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, handle_child, universal_pkg_installer::{par_install_language_dependencies_all_at_once, RequiredDependency}, - COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, JAVA_REPOSITORY_DIR, - MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + COURSIER_CACHE_DIR, DISABLE_NSJAIL, DISABLE_NUSER, JAVA_CACHE_DIR, + JAVA_REPOSITORY_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, }; use windmill_common::client::AuthedClient; @@ -653,11 +653,13 @@ async fn run<'a>( ) .await; - let mut cmd = Command::new(if cfg!(windows) { + let java_executable = if cfg!(windows) { "java" } else { JAVA_PATH.as_str() - }); + }; + + let mut cmd = build_command_with_isolation(java_executable, &[]); cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) @@ -703,7 +705,7 @@ async fn run<'a>( std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), ); } - start_child_process(cmd, "java", false).await? + start_child_process(cmd, java_executable, false).await? }; handle_child::handle_child( &job.id, @@ -836,7 +838,7 @@ fn wrap(inner_content: &str) -> Result { }) .collect_vec() .join(" "); - Ok(r#" + Ok(r#" package net.script; import com.fasterxml.jackson.databind.ObjectMapper; import java.io.FileInputStream; diff --git a/backend/windmill-worker/src/nu_executor.rs b/backend/windmill-worker/src/nu_executor.rs index 2497757d9d..917c5333ee 100644 --- a/backend/windmill-worker/src/nu_executor.rs +++ b/backend/windmill-worker/src/nu_executor.rs @@ -13,10 +13,11 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, - handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + handle_child, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, }; use windmill_common::client::AuthedClient; @@ -181,7 +182,7 @@ fn wrap(inner_content: &str) -> Result { .collect_vec() .join(" "); Ok( - r#" + r#" $env.config.table.mode = 'basic' def nullguard [ name: string ] { @@ -194,11 +195,11 @@ def nullguard [ name: string ] { # TODO: Probably needs rework in order for LSP to work def get_variable [ pat ] { let addr = $"($env.BASE_INTERNAL_URL)/api/w/($env.WM_WORKSPACE)/variables/get_value/($pat)" ; - http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in + http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in } def get_resource [ pat ] { let addr = $"($env.BASE_INTERNAL_URL)/api/w/($env.WM_WORKSPACE)/resources/get_value_interpolated/($pat)" ; - http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in + http get -H ["Authorization", $"Bearer ($env.WM_TOKEN)"] $addr | return $in } def 'main --wrapped' [] { @@ -285,11 +286,14 @@ async fn run<'a>( // let plugin_registry = format!("{job_dir}/plugin-registry"); // File::create(&plugin_registry).await?; // - let mut cmd = Command::new(if cfg!(windows) { + let nu_executable = if cfg!(windows) { "nu" } else { NU_PATH.as_str() - }); + }; + + let args = vec!["main.nu", "--wrapped"]; + let mut cmd = build_command_with_isolation(nu_executable, &args); cmd.env_clear() .current_dir(job_dir.to_owned()) .env("PATH", PATH_ENV.as_str()) @@ -297,20 +301,16 @@ async fn run<'a>( .envs(envs) .envs(reserved_variables) .envs(PROXY_ENVS.clone()) - .args(&[ - "main.nu", - "--wrapped", - // TODO(v1): - // "--plugins", - // &format!( - // "[{}]", - // plugins - // .into_iter() - // .map(|pl| format!("{NU_CACHE_DIR}/plugins/bin/nu_plugin_{pl}")) - // .collect_vec() - // .join(",") - // ), - ]) + // TODO(v1): + // "--plugins", + // &format!( + // "[{}]", + // plugins + // .into_iter() + // .map(|pl| format!("{NU_CACHE_DIR}/plugins/bin/nu_plugin_{pl}")) + // .collect_vec() + // .join(",") + // ), .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -324,7 +324,7 @@ async fn run<'a>( std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), ); } - start_child_process(cmd, "nu", false).await? + start_child_process(cmd, nu_executable, false).await? }; handle_child::handle_child( &job.id, diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index b272d631da..4ead742f63 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -16,11 +16,12 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, - read_result, start_child_process, OccupancyMetrics, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, + get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER, + NSJAIL_PATH, PHP_PATH, }; use windmill_common::client::AuthedClient; @@ -309,25 +310,22 @@ try {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let cmd = { - let script_path = format!("{job_dir}/wrapper.php"); + let script_path = format!("{job_dir}/wrapper.php"); + let args = vec![script_path.as_str()]; - let mut php_cmd = Command::new(&*PHP_PATH); - let args = vec![&script_path]; - php_cmd - .current_dir(job_dir) - .env_clear() - .envs(envs) - .envs(reserved_variables) - .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(args) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - php_cmd - }; - start_child_process(cmd, &*PHP_PATH, false).await? + let mut php_cmd = build_command_with_isolation(&*PHP_PATH, &args); + php_cmd + .current_dir(job_dir) + .env_clear() + .envs(envs) + .envs(reserved_variables) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .env("BASE_INTERNAL_URL", base_internal_url) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + start_child_process(php_cmd, &*PHP_PATH, false).await? }; handle_child( diff --git a/backend/windmill-worker/src/pwsh_executor.rs b/backend/windmill-worker/src/pwsh_executor.rs index 5c2b6f2e0a..b2a81234a3 100644 --- a/backend/windmill-worker/src/pwsh_executor.rs +++ b/backend/windmill-worker/src/pwsh_executor.rs @@ -23,12 +23,13 @@ lazy_static::lazy_static! { use crate::{ common::{ - build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, - OccupancyMetrics, + build_args_map, build_command_with_isolation, get_reserved_variables, read_file, + read_file_content, start_child_process, OccupancyMetrics, }, handle_child::handle_child, - DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, POWERSHELL_CACHE_DIR, - POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, TZ_ENV, + DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, + POWERSHELL_CACHE_DIR, POWERSHELL_PATH, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, PROXY_ENVS, + TZ_ENV, }; fn val_to_pwsh_param(v: serde_json::Value) -> String { @@ -81,14 +82,14 @@ $credentials = $null if ($hasPrivateRepo) { $repoName = "windmill-private-$jobId" $repoUri = "$privateRepoUrl" - + # Create PSCredential for authentication $username = "token" $patToken = ConvertTo-SecureString $privateRepoPat -AsPlainText -Force $credentials = New-Object System.Management.Automation.PSCredential($username, $patToken) - + Write-Host "Registering temporary repository: $repoName" - + # Remove repository if it already exists Unregister-PSResourceRepository -Name $repoName -ErrorAction SilentlyContinue Register-PSResourceRepository -Name $repoName -Uri $repoUri -Trusted @@ -99,7 +100,7 @@ try { foreach ($moduleRequest in $moduleRequests) { $moduleName = $moduleRequest.Name $requiredVersion = $moduleRequest.Version - + # Check if module is already installed with the required version (case-insensitive) $isInstalled = $false if ($requiredVersion) { @@ -107,32 +108,32 @@ try { } else { $isInstalled = $availableModules | Where-Object { $_.Name -eq $moduleName } } - + if (-not $isInstalled) { $moduleFound = $false - + # First try private repository if configured if ($hasPrivateRepo) { $findParams = @{ Name = $moduleName; Repository = $repoName; ErrorAction = 'SilentlyContinue'; Credential = $credentials } if ($requiredVersion) { $findParams.Version = $requiredVersion } - + $privateModule = Find-PSResource @findParams if ($privateModule) { $moduleFound = $true $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } Write-Host "Found module $moduleName$versionInfo in private repository, installing from there..." - + $saveParams = @{ Name = $moduleName; Path = $path; Repository = $repoName; Credential = $credentials } if ($requiredVersion) { $saveParams.Version = $requiredVersion } Save-PSResource @saveParams } } - + # If not found in private repo (or no private repo configured), try all repositories if (-not $moduleFound) { $versionInfo = if ($requiredVersion) { " version $requiredVersion" } else { "" } Write-Host "Installing module $moduleName$versionInfo from public repositories..." - + $saveParams = @{ Name = $moduleName; Path = $path; TrustRepository = $true } if ($requiredVersion) { $saveParams.Version = $requiredVersion } Save-PSResource @saveParams @@ -526,7 +527,6 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", start_child_process(cmd, NSJAIL_PATH.as_str(), false).await? } else { - let mut cmd = Command::new(POWERSHELL_PATH.as_str()); let cmd_args; #[cfg(unix)] @@ -539,6 +539,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", cmd_args = vec![r".\wrapper.ps1"]; } + let mut cmd = build_command_with_isolation(POWERSHELL_PATH.as_str(), &cmd_args); + cmd.current_dir(job_dir) .env_clear() .envs(envs) @@ -547,7 +549,6 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", .env("PATH", PATH_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(&cmd_args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 7ae80a97b6..bc84a2e19f 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -120,7 +120,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_SETTINGS; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_file, read_result, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, read_file, read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, handle_child::handle_child, @@ -835,9 +835,12 @@ mount {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { - let mut python_cmd = Command::new(&python_path); - let args = vec!["-u", "-m", "wrapper"]; + + let mut python_cmd = build_command_with_isolation( + &python_path, + &args, + ); python_cmd .current_dir(job_dir) .env_clear() @@ -847,7 +850,6 @@ mount {{ .env("TZ", TZ_ENV.as_str()) .env("BASE_INTERNAL_URL", base_internal_url) .env("HOME", HOME_ENV.as_str()) - .args(args) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); diff --git a/backend/windmill-worker/src/ruby_executor.rs b/backend/windmill-worker/src/ruby_executor.rs index cd25663889..0039e6e8c2 100644 --- a/backend/windmill-worker/src/ruby_executor.rs +++ b/backend/windmill-worker/src/ruby_executor.rs @@ -23,12 +23,13 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - create_args_and_out_file, get_reserved_variables, read_result, start_child_process, - OccupancyMetrics, + build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + read_result, start_child_process, OccupancyMetrics, }, handle_child::{self}, universal_pkg_installer::{par_install_language_dependencies_seq, RequiredDependency}, - DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, RUBY_CACHE_DIR, RUBY_REPOS, + DISABLE_NSJAIL, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, + RUBY_CACHE_DIR, RUBY_REPOS, }; lazy_static::lazy_static! { static ref RUBY_CONCURRENT_DOWNLOADS: usize = std::env::var("RUBY_CONCURRENT_DOWNLOADS").ok().map(|flag| flag.parse().unwrap_or(20)).unwrap_or(20); @@ -140,7 +141,7 @@ pub async fn prepare<'a>( File::create(format!("{}/inline.rb", &mini_wm_path)) .await? .write_all(&wrap( - r#" + r#" class GemfileProxy def initialize @gem_calls = [] @@ -213,7 +214,7 @@ end .await? .write_all( &wrap( - r##" + r##" require 'net/http' require 'uri' require 'json' @@ -223,17 +224,17 @@ def get_variable(path) base_url = ENV['BASE_INTERNAL_URL'] workspace = ENV['WM_WORKSPACE'] token = ENV['WM_TOKEN'] - + uri = URI("#{base_url}/api/w/#{workspace}/variables/get_value/#{path}") - + http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' - + request = Net::HTTP::Get.new(uri) request['Authorization'] = "Bearer #{token}" - + response = http.request(request) - + if response.code == '200' JSON.parse(response.body) else @@ -245,17 +246,17 @@ def get_resource(path) base_url = ENV['BASE_INTERNAL_URL'] workspace = ENV['WM_WORKSPACE'] token = ENV['WM_TOKEN'] - + uri = URI("#{base_url}/api/w/#{workspace}/resources/get_value_interpolated/#{path}") - + http = Net::HTTP.new(uri.host, uri.port) http.use_ssl = uri.scheme == 'https' - + request = Net::HTTP::Get.new(uri) request['Authorization'] = "Bearer #{token}" - + response = http.request(request) - + if response.code == '200' JSON.parse(response.body) else @@ -823,11 +824,14 @@ mount {{ ) .await; - let mut cmd = Command::new(if cfg!(windows) { + let ruby_executable = if cfg!(windows) { "ruby.exe" } else { RUBY_PATH.as_str() - }); + }; + + let args = vec!["main.rb"]; + let mut cmd = build_command_with_isolation(ruby_executable, &args); #[cfg(windows)] let rubylib = rubylib.replace(":", ";"); @@ -842,8 +846,7 @@ mount {{ .envs(PROXY_ENVS.clone()) .envs(envs); - cmd.args(&["main.rb"]) - .stdin(Stdio::null()) + cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -856,7 +859,7 @@ mount {{ std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), ); } - start_child_process(cmd, "ruby", false).await? + start_child_process(cmd, ruby_executable, false).await? }; handle_child::handle_child( &job.id, diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 6a05de459c..c43a7d2d97 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -19,7 +19,7 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ - check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, + build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, get_reserved_variables, read_result, start_child_process, OccupancyMetrics, }, handle_child::handle_child, @@ -567,7 +567,7 @@ pub async fn handle_rust_job( start_child_process(nsjail_cmd, NSJAIL_PATH.as_str(), false).await? } else { let compiled_executable_name = "./main"; - let mut run_rust = Command::new(compiled_executable_name); + let mut run_rust = build_command_with_isolation(compiled_executable_name, &[]); run_rust .current_dir(job_dir) .env_clear() diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index e58d521a5d..9b47aa5113 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -303,6 +303,128 @@ lazy_static::lazy_static! { .and_then(|x| x.parse::().ok()) .unwrap_or(true); + pub static ref ENABLE_UNSHARE_PID: bool = std::env::var("ENABLE_UNSHARE_PID") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); + + pub static ref UNSHARE_ISOLATION_FLAGS: String = { + std::env::var("UNSHARE_ISOLATION_FLAGS") + .unwrap_or_else(|_| "--user --map-root-user --pid --fork --mount-proc".to_string()) + }; + + pub static ref UNSHARE_PATH: Option = { + let flags = UNSHARE_ISOLATION_FLAGS.as_str(); + let mut test_cmd_args: Vec<&str> = flags.split_whitespace().collect(); + test_cmd_args.push("--"); + test_cmd_args.push("true"); + + let test_result = std::process::Command::new("unshare") + .args(&test_cmd_args) + .output(); + + match test_result { + Ok(output) if output.status.success() => { + tracing::info!("PID namespace isolation enabled. Flags: {}", flags); + Some("unshare".to_string()) + }, + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + + if *ENABLE_UNSHARE_PID { + panic!( + "ENABLE_UNSHARE_PID is set but unshare test failed.\n\ + Error: {}\n\ + Flags: {}\n\ + \n\ + Solutions:\n\ + • Check if user namespaces are enabled: 'sysctl kernel.unprivileged_userns_clone'\n\ + • For Docker: Requires 'privileged: true' in docker-compose for --mount-proc flag\n\ + • For Kubernetes: Requires 'privileged: true' in securityContext for --mount-proc flag\n\ + • Try different flags via UNSHARE_ISOLATION_FLAGS env var (remove --mount-proc if privileged mode not possible)\n\ + • Alternative: Use NSJAIL instead\n\ + • Disable: Set ENABLE_UNSHARE_PID=false", + stderr.trim(), + flags + ); + } + + tracing::warn!( + "unshare test failed: {}. Flags: {}. Set ENABLE_UNSHARE_PID=true to fail on error.", + stderr.trim(), + flags + ); + None + }, + Err(e) => { + if *ENABLE_UNSHARE_PID { + if e.kind() == std::io::ErrorKind::NotFound { + panic!( + "ENABLE_UNSHARE_PID is set but unshare binary not found.\n\ + Install util-linux package or set ENABLE_UNSHARE_PID=false" + ); + } else { + panic!( + "ENABLE_UNSHARE_PID is set but failed to test unshare: {}", + e + ); + } + } + + if e.kind() == std::io::ErrorKind::NotFound { + tracing::debug!("unshare binary not found"); + } else { + tracing::warn!("Failed to test unshare: {}", e); + } + None + } + } + }; + + pub static ref NSJAIL_AVAILABLE: Option = { + if *DISABLE_NSJAIL { + None + } else { + let nsjail_path = NSJAIL_PATH.as_str(); + + let test_result = std::process::Command::new(nsjail_path) + .arg("--help") + .output(); + + match test_result { + Ok(output) if output.status.success() => { + tracing::info!("NSJAIL sandboxing available at: {}", nsjail_path); + Some(nsjail_path.to_string()) + }, + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + tracing::warn!( + "nsjail test failed: {}. Jobs will run without nsjail sandboxing. \ + To enable nsjail: install nsjail binary or use windmill image with -nsjail suffix", + stderr.trim() + ); + None + }, + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + tracing::warn!( + "nsjail not found at '{}'. Jobs will run without nsjail sandboxing. \ + To enable nsjail: install nsjail binary or use windmill image with -nsjail suffix", + nsjail_path + ); + } else { + tracing::warn!( + "Failed to test nsjail at '{}': {}. Jobs will run without nsjail sandboxing.", + nsjail_path, + e + ); + } + None + } + } + } + }; + pub static ref KEEP_JOB_DIR: AtomicBool = AtomicBool::new(std::env::var("KEEP_JOB_DIR") .ok() .and_then(|x| x.parse::().ok()) @@ -981,6 +1103,19 @@ pub async fn run_worker( ); } + // Force UNSHARE_PATH initialization now to fail-fast if unshare doesn't work + // This ensures we panic at startup rather than lazily when first accessed during job execution + if *ENABLE_UNSHARE_PID { + // Access UNSHARE_PATH to trigger lazy_static initialization and test + let _ = &*UNSHARE_PATH; + + tracing::info!( + worker = %worker_name, hostname = %hostname, + "PID namespace isolation enabled via unshare with flags: {}", + UNSHARE_ISOLATION_FLAGS.as_str() + ); + } + let start_time = Instant::now(); let worker_dir = format!("{TMP_DIR}/{worker_name}"); diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index 366cc9cdf7..f105d8c415 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -143,6 +143,7 @@ async fn update_worker_ping_full_inner( memory: memory, memory_usage: get_worker_memory_usage(), wm_memory_usage: get_windmill_memory_usage(), + job_isolation: None, ping_type: PingType::MainLoop, }, ) @@ -171,6 +172,15 @@ pub async fn insert_ping( let vcpus = get_vcpus(); let memory = get_memory(); + // Determine job isolation method + let job_isolation = if crate::NSJAIL_AVAILABLE.is_some() { + Some("nsjail".to_string()) + } else if *crate::ENABLE_UNSHARE_PID && crate::UNSHARE_PATH.is_some() { + Some("unshare".to_string()) + } else { + Some("none".to_string()) + }; + match db { Connection::Sql(db) => { insert_ping_query( @@ -183,6 +193,7 @@ pub async fn insert_ping( windmill_common::utils::GIT_VERSION, vcpus, memory, + job_isolation, db, ) .await?; @@ -209,6 +220,7 @@ pub async fn insert_ping( memory: memory, memory_usage: get_worker_memory_usage(), wm_memory_usage: get_windmill_memory_usage(), + job_isolation, ping_type: PingType::Initial, }, ) @@ -231,6 +243,15 @@ pub async fn update_worker_ping_from_job( let occupancy_rate_15s = occupancy.as_ref().and_then(|x| x.occupancy_rate_15s); let occupancy_rate_5m = occupancy.as_ref().and_then(|x| x.occupancy_rate_5m); let occupancy_rate_30m = occupancy.as_ref().and_then(|x| x.occupancy_rate_30m); + + let job_isolation = if crate::NSJAIL_AVAILABLE.is_some() { + Some("nsjail".to_string()) + } else if *crate::ENABLE_UNSHARE_PID && crate::UNSHARE_PATH.is_some() { + Some("unshare".to_string()) + } else { + Some("none".to_string()) + }; + match conn.clone() { Connection::Sql(ref db) => { update_worker_ping_from_job_query( @@ -243,6 +264,7 @@ pub async fn update_worker_ping_from_job( occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, + job_isolation, db, ) .await?; @@ -270,6 +292,7 @@ pub async fn update_worker_ping_from_job( occupancy_rate_15s: occupancy_rate_15s, occupancy_rate_5m: occupancy_rate_5m, occupancy_rate_30m: occupancy_rate_30m, + job_isolation, }, ) .await?; diff --git a/docker-compose.yml b/docker-compose.yml index ea67377ffc..a6d96aaec9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,10 +60,16 @@ services: memory: 2048M # for GB, use syntax '2Gi' restart: unless-stopped + # Uncomment to enable PID namespace isolation (recommended for security) + # Requires privileged mode for --mount-proc flag + # See: https://www.windmill.dev/docs/advanced/security_isolation + # privileged: true environment: - DATABASE_URL=${DATABASE_URL} - MODE=worker - WORKER_GROUP=default + # Uncomment to enable PID namespace isolation (requires privileged: true above) + # - ENABLE_UNSHARE_PID=true depends_on: db: condition: service_healthy @@ -89,12 +95,18 @@ services: memory: 2048M # for GB, use syntax '2Gi' restart: unless-stopped + # Uncomment to enable PID namespace isolation (recommended for security) + # Requires privileged mode for --mount-proc flag + # See: https://www.windmill.dev/docs/advanced/security_isolation + # privileged: true environment: - DATABASE_URL=${DATABASE_URL} - MODE=worker - WORKER_GROUP=native - NUM_WORKERS=8 - SLEEP_QUEUE=200 + # Uncomment to enable PID namespace isolation (requires privileged: true above) + # - ENABLE_UNSHARE_PID=true depends_on: db: condition: service_healthy @@ -113,10 +125,16 @@ services: # memory: 2048M # # for GB, use syntax '2Gi' # restart: unless-stopped + # # Uncomment to enable PID namespace isolation (recommended for security) + # # Requires privileged mode for --mount-proc flag + # # See: https://www.windmill.dev/docs/advanced/security_isolation + # # privileged: true # environment: # - DATABASE_URL=${DATABASE_URL} # - MODE=worker # - WORKER_GROUP=reports + # # Uncomment to enable PID namespace isolation (requires privileged: true above) + # # - ENABLE_UNSHARE_PID=true # depends_on: # db: # condition: service_healthy diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 4457f536c6..d22e8c8539 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -1,6 +1,8 @@