From 52f39a0e04606dec4e7dfbce44b328015e5ce12b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 3 Jun 2026 12:48:14 +0000 Subject: [PATCH] feat(worker): rootless podman container runtime for docker-mode jobs Add a per-worker-group `container_runtime` option. When set to "podman", the worker starts a rootless, daemonless `podman system service` on a unix socket and points DOCKER_HOST at it, so docker-mode jobs (`# docker` bash scripts) run against a Docker-API-compatible daemon without a privileged dind sidecar or the host Docker socket. bollard's existing handle_docker_job talks to it unchanged (validated: inspect/wait/logs/stats/kill/stop/remove against podman 5.8.2). - windmill-common: container_runtime field on WorkerConfigOpt/WorkerConfig, loaded from group config or CONTAINER_RUNTIME env. - windmill-worker: start_container_runtime_maybe() spawns the rootless podman service once per process (i_worker==1), waits for the socket, sets DOCKER_HOST. Respects an existing DOCKER_HOST (legacy dind) and degrades with a clear log if the podman binary is absent (use a *-full image or an init script). - monitor: restart the worker (killpill) when container_runtime changes. - api-configs: allow container_runtime in CE config updates. Requires rootless podman with cgroup v2 delegation for memory monitoring; podman ships in the *-full images (next commit). Co-Authored-By: Claude Opus 4.8 (1M context) --- backend/src/monitor.rs | 8 +++ backend/windmill-api-configs/src/lib.rs | 28 +++++--- backend/windmill-common/src/worker.rs | 16 ++++- backend/windmill-worker/src/worker.rs | 91 +++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 12 deletions(-) diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 13f52037e7..5567eaae11 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -315,6 +315,9 @@ pub async fn initial_load( additional_python_paths: None, pip_local_dependencies: None, native_mode, + container_runtime: std::env::var("CONTAINER_RUNTIME") + .ok() + .filter(|x| !x.is_empty()), })); } } @@ -3002,6 +3005,11 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b let _ = tx.send(); } + if wc.container_runtime != config.container_runtime { + tracing::info!("Container runtime config changed, sending killpill. Expecting to be restarted by supervisor."); + let _ = tx.send(); + } + if wc.cache_clear != config.cache_clear { tracing::info!("Cache clear changed, sending killpill. Expecting to be restarted by supervisor."); let _ = tx.send(); diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index e18afc35c8..769ceda18b 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -63,10 +63,12 @@ async fn list_worker_groups( authed: ApiAuthed, Extension(db): Extension, ) -> error::JsonResult> { - let mut configs_raw = - sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'") - .fetch_all(&db) - .await?; + let mut configs_raw = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name LIKE 'worker__%'" + ) + .fetch_all(&db) + .await?; // Remove the 'worker__' prefix from all config names for config in configs_raw.iter_mut() { if let Some(name) = &config.name { @@ -119,10 +121,14 @@ async fn get_config( ) -> error::JsonResult> { require_devops_role(&db, &authed.email).await?; - let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name) - .fetch_optional(&db) - .await? - .map(|c| c.config); + let config = sqlx::query_as!( + Config, + "SELECT name, config FROM config WHERE name = $1", + name + ) + .fetch_optional(&db) + .await? + .map(|c| c.config); Ok(Json(config)) } @@ -137,12 +143,14 @@ async fn update_config( #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { - // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode + // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode, + // and container_runtime serde_json::json!({ "worker_tags": config.get("worker_tags"), "cache_clear": config.get("cache_clear"), "init_bash": config.get("init_bash"), - "native_mode": config.get("native_mode") + "native_mode": config.get("native_mode"), + "container_runtime": config.get("container_runtime") }) } else { config diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 92ebf08477..20ffe6921e 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -234,6 +234,7 @@ lazy_static::lazy_static! { pip_local_dependencies: Default::default(), env_vars: Default::default(), native_mode: false, + container_runtime: None, }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -1960,6 +1961,10 @@ pub async fn load_worker_config( .or_else(|| load_additional_python_paths_from_env()), env_vars: resolved_env_vars, native_mode, + container_runtime: config + .container_runtime + .or_else(|| std::env::var("CONTAINER_RUNTIME").ok()) + .and_then(|x| if x.is_empty() { None } else { Some(x) }), }) } @@ -2049,6 +2054,11 @@ pub struct WorkerConfigOpt { pub env_vars_static: Option>, pub env_vars_allowlist: Option>, pub native_mode: Option, + /// Container runtime for docker-mode jobs on this worker group. When set to + /// "podman", the worker starts a rootless podman service and points + /// DOCKER_HOST at it (see start_container_runtime). None = no managed runtime + /// (legacy dind/host-socket via DOCKER_HOST/socket still works if present). + pub container_runtime: Option, } impl Default for WorkerConfigOpt { @@ -2067,6 +2077,7 @@ impl Default for WorkerConfigOpt { env_vars_static: Default::default(), env_vars_allowlist: Default::default(), native_mode: Default::default(), + container_runtime: Default::default(), } } } @@ -2085,12 +2096,13 @@ pub struct WorkerConfig { pub pip_local_dependencies: Option>, pub env_vars: HashMap, pub native_mode: bool, + pub container_runtime: Option, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode) + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, container_runtime: {:?} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, self.container_runtime) } } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7fb0dee9f5..97dbb12b6f 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -2244,6 +2244,10 @@ pub async fn run_worker( ) = (HashMap::new(), vec![]); if i_worker == 1 { + // Start the configured container runtime (rootless podman) before any job + // runs, so docker-mode jobs have a Docker-API-compatible daemon and + // DOCKER_HOST is set process-wide. No-op unless container_runtime is set. + start_container_runtime_maybe().await; // Initialize runtime asset inserter for batched database inserts if let Connection::Sql(db) = conn { init_runtime_asset_loop(db.clone(), killpill_rx.resubscribe()); @@ -3235,6 +3239,93 @@ pub async fn run_worker( tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } +// Holds the managed podman service child so it is not dropped (and thus not +// reaped) for the lifetime of the worker process. +static PODMAN_SERVICE: std::sync::Mutex> = + std::sync::Mutex::new(None); + +/// Start the container runtime configured on this worker group (currently only +/// rootless "podman") so docker-mode jobs have a Docker-API-compatible daemon to +/// talk to — without a privileged dind sidecar or the host Docker socket. Spawns +/// a rootless `podman system service` on a unix socket and points DOCKER_HOST at +/// it, which both `connect_docker()` (bollard) and the docker CLI forwarded to +/// scripts then use. No-op unless `container_runtime` is set on the group. +async fn start_container_runtime_maybe() { + let runtime = match WORKER_CONFIG.load().container_runtime.clone() { + Some(r) => r, + None => return, + }; + if runtime != "podman" { + tracing::error!( + "Unknown container_runtime '{runtime}' (expected 'podman'); not starting a container runtime." + ); + return; + } + // Respect an explicitly-provided DOCKER_HOST (e.g. a legacy dind/host socket). + if std::env::var("DOCKER_HOST").is_ok() { + tracing::info!( + "container_runtime=podman but DOCKER_HOST is already set; using the existing daemon and not starting the managed podman runtime." + ); + return; + } + let podman_ok = tokio::process::Command::new("podman") + .arg("--version") + .output() + .await + .map(|o| o.status.success()) + .unwrap_or(false); + if !podman_ok { + tracing::error!( + "container_runtime=podman but the 'podman' binary was not found on this worker. Use a windmill *-full image, or install podman via a worker-group init script. Docker-mode jobs will fail until podman is available." + ); + return; + } + + let sock_dir = "/tmp/windmill"; + let _ = tokio::fs::create_dir_all(sock_dir).await; + let sock_path = format!("{sock_dir}/podman.sock"); + let _ = tokio::fs::remove_file(&sock_path).await; // clear any stale socket + let docker_host = format!("unix://{sock_path}"); + + match tokio::process::Command::new("podman") + .args(["system", "service", "--time=0", &docker_host]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + { + Ok(child) => { + if let Ok(mut guard) = PODMAN_SERVICE.lock() { + *guard = Some(child); + } + // Wait (up to ~10s) for the rootless podman service socket to appear + // so the first docker job doesn't race startup. + let mut ready = false; + for _ in 0..50 { + if tokio::fs::metadata(&sock_path).await.is_ok() { + ready = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + } + std::env::set_var("DOCKER_HOST", &docker_host); + if ready { + tracing::info!( + "Started rootless podman container runtime, DOCKER_HOST={docker_host}" + ); + } else { + tracing::warn!( + "Started rootless podman container runtime but socket {sock_path} did not appear within 10s; DOCKER_HOST set to {docker_host} anyway." + ); + } + } + Err(e) => { + tracing::error!( + "Failed to start podman container runtime: {e:#}. Docker-mode jobs will fail." + ); + } + } +} + async fn queue_init_bash_maybe<'c>( conn: &Connection, same_worker_tx: SameWorkerSender,