feat(worker): rootless podman container runtime for docker-mode jobs

Add a per-worker-group `container_runtime` option. When set to "podman", the
worker starts a rootless, daemonless `podman system service` on a unix socket
and points DOCKER_HOST at it, so docker-mode jobs (`# docker` bash scripts) run
against a Docker-API-compatible daemon without a privileged dind sidecar or the
host Docker socket. bollard's existing handle_docker_job talks to it unchanged
(validated: inspect/wait/logs/stats/kill/stop/remove against podman 5.8.2).

- windmill-common: container_runtime field on WorkerConfigOpt/WorkerConfig,
  loaded from group config or CONTAINER_RUNTIME env.
- windmill-worker: start_container_runtime_maybe() spawns the rootless podman
  service once per process (i_worker==1), waits for the socket, sets DOCKER_HOST.
  Respects an existing DOCKER_HOST (legacy dind) and degrades with a clear log if
  the podman binary is absent (use a *-full image or an init script).
- monitor: restart the worker (killpill) when container_runtime changes.
- api-configs: allow container_runtime in CE config updates.

Requires rootless podman with cgroup v2 delegation for memory monitoring;
podman ships in the *-full images (next commit).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

backend/src/monitor.rs

@@ -315,6 +315,9 @@ pub async fn initial_load(
                     additional_python_paths: None,
                     pip_local_dependencies: None,
                     native_mode,
+                    container_runtime: std::env::var("CONTAINER_RUNTIME")
+                        .ok()
+                        .filter(|x| !x.is_empty()),
                 }));
             }
         }
@@ -3002,6 +3005,11 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b
                     let _ = tx.send();
                 }
 
+                if wc.container_runtime != config.container_runtime {
+                    tracing::info!("Container runtime config changed, sending killpill. Expecting to be restarted by supervisor.");
+                    let _ = tx.send();
+                }
+
                 if wc.cache_clear != config.cache_clear {
                     tracing::info!("Cache clear changed, sending killpill. Expecting to be restarted by supervisor.");
                     let _ = tx.send();

backend/windmill-api-configs/src/lib.rs

@@ -63,10 +63,12 @@ async fn list_worker_groups(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
 ) -> error::JsonResult<Vec<Config>> {
-    let mut configs_raw =
-        sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'")
-            .fetch_all(&db)
-            .await?;
+    let mut configs_raw = sqlx::query_as!(
+        Config,
+        "SELECT name, config FROM config WHERE name LIKE 'worker__%'"
+    )
+    .fetch_all(&db)
+    .await?;
     // Remove the 'worker__' prefix from all config names
     for config in configs_raw.iter_mut() {
         if let Some(name) = &config.name {
@@ -119,10 +121,14 @@ async fn get_config(
 ) -> error::JsonResult<Option<serde_json::Value>> {
     require_devops_role(&db, &authed.email).await?;
 
-    let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name)
-        .fetch_optional(&db)
-        .await?
-        .map(|c| c.config);
+    let config = sqlx::query_as!(
+        Config,
+        "SELECT name, config FROM config WHERE name = $1",
+        name
+    )
+    .fetch_optional(&db)
+    .await?
+    .map(|c| c.config);
 
     Ok(Json(config))
 }
@@ -137,12 +143,14 @@ async fn update_config(
 
     #[cfg(not(feature = "enterprise"))]
     let config = if name.starts_with("worker__") {
-        // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode
+        // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode,
+        // and container_runtime
         serde_json::json!({
             "worker_tags": config.get("worker_tags"),
             "cache_clear": config.get("cache_clear"),
             "init_bash": config.get("init_bash"),
-            "native_mode": config.get("native_mode")
+            "native_mode": config.get("native_mode"),
+            "container_runtime": config.get("container_runtime")
         })
     } else {
         config

backend/windmill-common/src/worker.rs

@@ -234,6 +234,7 @@ lazy_static::lazy_static! {
         pip_local_dependencies: Default::default(),
         env_vars: Default::default(),
         native_mode: false,
+        container_runtime: None,
     });
 
     pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap<Vec<String>> = arc_swap::ArcSwap::from_pointee(vec![]);
@@ -1960,6 +1961,10 @@ pub async fn load_worker_config(
             .or_else(|| load_additional_python_paths_from_env()),
         env_vars: resolved_env_vars,
         native_mode,
+        container_runtime: config
+            .container_runtime
+            .or_else(|| std::env::var("CONTAINER_RUNTIME").ok())
+            .and_then(|x| if x.is_empty() { None } else { Some(x) }),
     })
 }
 
@@ -2049,6 +2054,11 @@ pub struct WorkerConfigOpt {
     pub env_vars_static: Option<HashMap<String, String>>,
     pub env_vars_allowlist: Option<Vec<String>>,
     pub native_mode: Option<bool>,
+    /// Container runtime for docker-mode jobs on this worker group. When set to
+    /// "podman", the worker starts a rootless podman service and points
+    /// DOCKER_HOST at it (see start_container_runtime). None = no managed runtime
+    /// (legacy dind/host-socket via DOCKER_HOST/socket still works if present).
+    pub container_runtime: Option<String>,
 }
 
 impl Default for WorkerConfigOpt {
@@ -2067,6 +2077,7 @@ impl Default for WorkerConfigOpt {
             env_vars_static: Default::default(),
             env_vars_allowlist: Default::default(),
             native_mode: Default::default(),
+            container_runtime: Default::default(),
         }
     }
 }
@@ -2085,12 +2096,13 @@ pub struct WorkerConfig {
     pub pip_local_dependencies: Option<Vec<String>>,
     pub env_vars: HashMap<String, String>,
     pub native_mode: bool,
+    pub container_runtime: Option<String>,
 }
 
 impl std::fmt::Debug for WorkerConfig {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?} }}",
-        self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::<Vec<String>>().join(", "), self.native_mode)
+        write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, container_runtime: {:?} }}",
+        self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::<Vec<String>>().join(", "), self.native_mode, self.container_runtime)
     }
 }
 

backend/windmill-worker/src/worker.rs

@@ -2244,6 +2244,10 @@ pub async fn run_worker(
     ) = (HashMap::new(), vec![]);
 
     if i_worker == 1 {
+        // Start the configured container runtime (rootless podman) before any job
+        // runs, so docker-mode jobs have a Docker-API-compatible daemon and
+        // DOCKER_HOST is set process-wide. No-op unless container_runtime is set.
+        start_container_runtime_maybe().await;
         // Initialize runtime asset inserter for batched database inserts
         if let Connection::Sql(db) = conn {
             init_runtime_asset_loop(db.clone(), killpill_rx.resubscribe());
@@ -3235,6 +3239,93 @@ pub async fn run_worker(
     tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed);
 }
 
+// Holds the managed podman service child so it is not dropped (and thus not
+// reaped) for the lifetime of the worker process.
+static PODMAN_SERVICE: std::sync::Mutex<Option<tokio::process::Child>> =
+    std::sync::Mutex::new(None);
+
+/// Start the container runtime configured on this worker group (currently only
+/// rootless "podman") so docker-mode jobs have a Docker-API-compatible daemon to
+/// talk to — without a privileged dind sidecar or the host Docker socket. Spawns
+/// a rootless `podman system service` on a unix socket and points DOCKER_HOST at
+/// it, which both `connect_docker()` (bollard) and the docker CLI forwarded to
+/// scripts then use. No-op unless `container_runtime` is set on the group.
+async fn start_container_runtime_maybe() {
+    let runtime = match WORKER_CONFIG.load().container_runtime.clone() {
+        Some(r) => r,
+        None => return,
+    };
+    if runtime != "podman" {
+        tracing::error!(
+            "Unknown container_runtime '{runtime}' (expected 'podman'); not starting a container runtime."
+        );
+        return;
+    }
+    // Respect an explicitly-provided DOCKER_HOST (e.g. a legacy dind/host socket).
+    if std::env::var("DOCKER_HOST").is_ok() {
+        tracing::info!(
+            "container_runtime=podman but DOCKER_HOST is already set; using the existing daemon and not starting the managed podman runtime."
+        );
+        return;
+    }
+    let podman_ok = tokio::process::Command::new("podman")
+        .arg("--version")
+        .output()
+        .await
+        .map(|o| o.status.success())
+        .unwrap_or(false);
+    if !podman_ok {
+        tracing::error!(
+            "container_runtime=podman but the 'podman' binary was not found on this worker. Use a windmill *-full image, or install podman via a worker-group init script. Docker-mode jobs will fail until podman is available."
+        );
+        return;
+    }
+
+    let sock_dir = "/tmp/windmill";
+    let _ = tokio::fs::create_dir_all(sock_dir).await;
+    let sock_path = format!("{sock_dir}/podman.sock");
+    let _ = tokio::fs::remove_file(&sock_path).await; // clear any stale socket
+    let docker_host = format!("unix://{sock_path}");
+
+    match tokio::process::Command::new("podman")
+        .args(["system", "service", "--time=0", &docker_host])
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .spawn()
+    {
+        Ok(child) => {
+            if let Ok(mut guard) = PODMAN_SERVICE.lock() {
+                *guard = Some(child);
+            }
+            // Wait (up to ~10s) for the rootless podman service socket to appear
+            // so the first docker job doesn't race startup.
+            let mut ready = false;
+            for _ in 0..50 {
+                if tokio::fs::metadata(&sock_path).await.is_ok() {
+                    ready = true;
+                    break;
+                }
+                tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+            }
+            std::env::set_var("DOCKER_HOST", &docker_host);
+            if ready {
+                tracing::info!(
+                    "Started rootless podman container runtime, DOCKER_HOST={docker_host}"
+                );
+            } else {
+                tracing::warn!(
+                    "Started rootless podman container runtime but socket {sock_path} did not appear within 10s; DOCKER_HOST set to {docker_host} anyway."
+                );
+            }
+        }
+        Err(e) => {
+            tracing::error!(
+                "Failed to start podman container runtime: {e:#}. Docker-mode jobs will fail."
+            );
+        }
+    }
+}
+
 async fn queue_init_bash_maybe<'c>(
     conn: &Connection,
     same_worker_tx: SameWorkerSender,