refactor(worker): drop docker image storage cap; scope rootless netns per-job

Remove the `docker_image_storage_size_mb` instance setting and the polling
storage monitor entirely. Now that `# docker` jobs are refused under nsjail
(the per-job podman daemon runs outside the sandbox), the asymmetry that
justified a docker-specific disk cap is gone: a normal job can already exhaust
host disk in non-nsjail modes, so a docker-only cap was inconsistent. Docker
jobs now use disk like any other job — bound it at the infra level.

Also scope the rootless slirp4netns network-backend override to the per-job
podman instance via a job-scoped `$HOME` containers.conf instead of a global
`/etc/containers` drop-in, so rootful podman elsewhere in the *-full image is
unaffected (flagged by cubic).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

README.md

@@ -207,6 +207,44 @@ docker compose up -d
 
 Go to http://localhost - default credentials: `admin@windmill.dev` / `changeme`
 
+> [!NOTE]
+> To run `# docker` scripts (bash scripts with the `# docker` annotation): just give a
+> worker a `*-full` image (ships podman). `# docker` scripts are auto-tagged `docker`
+> (served by default workers out of the box; route them to a dedicated/bigger group
+> with `WORKER_TAGS=docker`). On a worker with **no Docker daemon provided** (no
+> `DOCKER_HOST`, no mounted `/var/run/docker.sock`), Windmill runs each docker job in
+> its own ephemeral **rootless podman**, torn down with the job — no privileged daemon,
+> no host socket, scripts unchanged. (Docker jobs use disk like any other job — bound it
+> at the infra level, e.g. a sized `/tmp/windmill` volume.) On old kernels (<5.13)
+> also expose `/dev/fuse`. The `*-full` image is also the batteries-included runtime —
+> on top of the base (TS/Bun/Deno, Python, Go) it adds Java, .NET, Ruby, R, Rust,
+> Ansible, and Nushell (plus Oracle/Kerberos in EE). See the default `windmill_worker`
+> notes in
+> [docker-compose.yml](./docker-compose.yml). To use an external/host Docker daemon
+> instead (legacy), provide `DOCKER_HOST` or mount `/var/run/docker.sock`.
+>
+> **Mixed fleets:** `docker` is a default tag, so a base/slim default worker (no podman)
+> can pick up a docker job and fail it ("no Docker daemon… use a `*-full` image"). If not
+> all your default workers use a `*-full` image, route docker jobs to a dedicated
+> `*-full` group with `WORKER_TAGS=docker` rather than relying on the default tag.
+>
+> **Security:** keep the docker worker running as **root** (the default — don't set a
+> non-root `user`): windmill drops the per-job podman to a non-root uid itself, so
+> containers are rootless **and** the worker's own secrets in `/proc` (`DATABASE_URL`
+> etc.) stay root-owned — unreadable by a `docker run --pid=host` container. What this
+> does **not** do is confine the container's *filesystem* view: the daemon runs outside
+> the job's nsjail sandbox, so a `# docker` script can still `docker run -v <worker path>`
+> to reach **world-readable** host state: other concurrent job dirs' world-readable files,
+> the dependency cache, and world-readable `/proc` (e.g. `-v /proc` → process `cmdline`s /
+> args) — though **not** `0400` files like `/proc/<pid>/environ`, so the worker's env
+> secrets stay protected. (Corollary: don't pass secrets as command-line args; windmill
+> uses env, not args.) So treat docker-capable workers as a **trusted-tenant** capability
+> (a big improvement over privileged dind, but not a full sandbox) and prefer
+> dedicated/per-workspace docker workers on shared multi-tenant fleets.
+> If you enable **nsjail** (it's off by default), per-job podman is **not** auto-provided
+> for docker jobs — nsjail means "fully sandbox jobs" and the daemon runs outside the jail,
+> so docker jobs then require an explicit `DOCKER_HOST` or a separate non-nsjail worker group.
+
 **Using an external database**: Set `DATABASE_URL` in `.env` to point to your managed Postgres (AWS RDS, GCP Cloud SQL, Azure, Neon, etc.) and set db replicas to 0.
 
 ### Kubernetes (Helm charts)

backend/Cargo.lock

@@ -15620,6 +15620,7 @@ dependencies = [
  "itertools 0.14.0",
  "jsonwebtoken 8.3.0",
  "lazy_static",
+ "libc",
  "libffi-sys",
  "libloading",
  "mappable-rc",

backend/windmill-api-configs/src/lib.rs

@@ -63,10 +63,12 @@ async fn list_worker_groups(
     authed: ApiAuthed,
     Extension(db): Extension<DB>,
 ) -> error::JsonResult<Vec<Config>> {
-    let mut configs_raw =
-        sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name LIKE 'worker__%'")
-            .fetch_all(&db)
-            .await?;
+    let mut configs_raw = sqlx::query_as!(
+        Config,
+        "SELECT name, config FROM config WHERE name LIKE 'worker__%'"
+    )
+    .fetch_all(&db)
+    .await?;
     // Remove the 'worker__' prefix from all config names
     for config in configs_raw.iter_mut() {
         if let Some(name) = &config.name {
@@ -119,10 +121,14 @@ async fn get_config(
 ) -> error::JsonResult<Option<serde_json::Value>> {
     require_devops_role(&db, &authed.email).await?;
 
-    let config = sqlx::query_as!(Config, "SELECT name, config FROM config WHERE name = $1", name)
-        .fetch_optional(&db)
-        .await?
-        .map(|c| c.config);
+    let config = sqlx::query_as!(
+        Config,
+        "SELECT name, config FROM config WHERE name = $1",
+        name
+    )
+    .fetch_optional(&db)
+    .await?
+    .map(|c| c.config);
 
     Ok(Json(config))
 }

backend/windmill-api-scripts/src/scripts.rs

@@ -894,6 +894,19 @@ async fn create_script_internal<'c>(
     }
     check_scopes(&authed, || format!("scripts:write:{}", ns.path))?;
 
+    // Auto-route Bash `# docker` scripts to the `docker` tag (part of DEFAULT_TAGS)
+    // when no explicit tag is set, so they can be directed to docker-capable / bigger
+    // workers. Done here — before no-op detection and the insert — so an unchanged
+    // redeploy is still correctly detected as a no-op. Tag-only mechanism mirroring
+    // the routing half of bunnative/nativets (the bash executor handles `# docker`
+    // at runtime, so no distinct ScriptLang is needed).
+    if ns.tag.as_deref().map_or(true, |t| t.is_empty())
+        && ns.language == ScriptLang::Bash
+        && windmill_common::worker::BashAnnotations::parse(&ns.content).docker
+    {
+        ns.tag = Some(windmill_common::worker::DOCKER_BASH_TAG.to_string());
+    }
+
     guard_script_from_debounce_data(&ns).await?;
 
     let codebase = ns.codebase.as_ref();

backend/windmill-common/src/worker.rs

@@ -169,6 +169,10 @@ lazy_static::lazy_static! {
         "python3".to_string(),
         "go".to_string(),
         "bash".to_string(),
+        // Bash `# docker` scripts are auto-tagged `docker` (see DOCKER_BASH_TAG /
+        // BashAnnotations) so they can be routed to docker-capable workers; in
+        // DEFAULT_TAGS so default workers serve them out of the box.
+        DOCKER_BASH_TAG.to_string(),
         "powershell".to_string(),
         "nativets".to_string(),
         "mysql".to_string(),
@@ -859,6 +863,15 @@ pub struct BashAnnotations {
     pub sandbox: bool,
 }
 
+/// Tag assigned to Bash scripts carrying the `# docker` annotation so docker
+/// jobs can be routed to docker-capable / bigger workers. This is a tag-only
+/// mechanism (no distinct ScriptLang — the bash executor handles the annotation
+/// at runtime), mirroring the routing half of `bunnative`/`nativets`. It is part
+/// of DEFAULT_TAGS so default workers serve it out of the box; derived at script
+/// create (stored on `script.tag`) and at preview push when no explicit tag is
+/// set. Callers gate on `language == Bash`.
+pub const DOCKER_BASH_TAG: &str = "docker";
+
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum SqlResultCollectionStrategy {
     LastStatementAllRows,

backend/windmill-queue/src/jobs.rs

@@ -5744,6 +5744,14 @@ async fn push_inner<'c, 'd>(
                             } else {
                                 ScriptLang::Nativets.as_str()
                             }
+                        } else if x == &ScriptLang::Bash
+                            && raw_code.as_deref().is_some_and(|c| {
+                                windmill_common::worker::BashAnnotations::parse(c).docker
+                            })
+                        {
+                            // Bash `# docker` previews/raw runs route to the `docker`
+                            // tag (deployed scripts get it stored at create time).
+                            windmill_common::worker::DOCKER_BASH_TAG
                         } else {
                             x.as_str()
                         };

backend/windmill-worker/Cargo.toml

@@ -115,6 +115,7 @@ pem = { workspace = true, optional = true }
 rsa = { workspace = true, optional = true }
 urlencoding.workspace = true
 nix.workspace = true
+libc = "0.2"
 bytes.workspace = true
 reqwest.workspace = true
 reqwest-middleware.workspace = true

backend/windmill-worker/nsjail/run.bash.config.proto

@@ -70,6 +70,9 @@ mount {
 
 {TMP_MOUNT_BLOCK}
 
+# Per-job rootless podman socket (docker jobs under nsjail); empty otherwise.
+{DOCKER_SOCK_MOUNT}
+
 mount {
     src: "{JOB_DIR}/main.sh"
     dst: "/tmp/main.sh"

backend/windmill-worker/src/bash_executor.rs

@@ -41,8 +41,8 @@ use crate::handle_child::run_future_with_polling_update_job_poller;
 use crate::{
     common::{
         build_args_map, build_command_with_isolation, get_reserved_variables, read_file,
-        read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block, start_child_process,
-        OccupancyMetrics, DEV_CONF_NSJAIL,
+        read_file_content, resolve_nsjail_timeout, resolve_nsjail_tmp_mount_block,
+        start_child_process, OccupancyMetrics, DEV_CONF_NSJAIL,
     },
     get_proxy_envs_for_lang,
     handle_child::handle_child,
@@ -180,17 +180,6 @@ exit $exit_status
     let _ = write_file(job_dir, "result.out", "")?;
     let _ = write_file(job_dir, "result2.out", "")?;
 
-    // Forward DOCKER_HOST to the bash script when in docker mode so the docker CLI
-    // connects to the right daemon (e.g. a dind sidecar instead of /var/run/docker.sock)
-    let docker_envs: Vec<(&str, String)> = if annotation.docker {
-        ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"]
-            .iter()
-            .filter_map(|k| std::env::var(k).ok().map(|v| (*k, v)))
-            .collect()
-    } else {
-        vec![]
-    };
-
     // Check if this is a regular job (not init or periodic script)
     // Init/periodic scripts need full system access without isolation
     let is_regular_job = job
@@ -203,6 +192,98 @@ exit $exit_status
 
     // Use nsjail if globally enabled OR if script has #sandbox annotation
     let nsjail = (is_sandboxing_enabled() || annotation.sandbox) && is_regular_job;
+
+    // Docker runtime selection: if a Docker daemon is already provided — DOCKER_HOST
+    // set, or the host socket mounted at /var/run/docker.sock — use it (legacy,
+    // backwards compatible). DOCKER_HOST counts in all modes (a tcp:// dind stays
+    // reachable through the jail since the sandbox doesn't isolate the network); a
+    // mounted /var/run/docker.sock only counts when NOT under nsjail (the bash nsjail
+    // proto never bind-mounts it into the jail, so it's unreachable there).
+    //
+    // Otherwise, for a `# docker` job, start a per-job rootless podman — BUT NOT under
+    // nsjail. The per-job daemon runs OUTSIDE the jail and a `# docker` script can
+    // bind-mount worker-visible paths through it, bypassing the jail's filesystem
+    // isolation. Auto-providing that under nsjail would be a surprising hole: enabling
+    // nsjail is an explicit "fully sandbox jobs" signal, so we refuse instead and make
+    // the operator opt into docker explicitly (a provided DOCKER_HOST, or a worker
+    // group without nsjail). Requires podman in the image (the *-full images).
+    #[cfg(feature = "dind")]
+    let docker_daemon_provided = std::env::var("DOCKER_HOST").is_ok()
+        || (!nsjail && std::path::Path::new("/var/run/docker.sock").exists());
+    #[cfg(feature = "dind")]
+    let per_job_podman: Option<PerJobPodman> = if annotation.docker && !docker_daemon_provided {
+        if nsjail {
+            return Err(Error::ExecutionErr(
+                "`# docker` jobs are not given an automatic container runtime under nsjail \
+                 sandboxing: the per-job podman would run outside the sandbox and a docker \
+                 script could bind-mount worker-visible paths through it, weakening the \
+                 isolation you enabled nsjail for. Run docker jobs on a worker group without \
+                 nsjail, or provide a Docker daemon explicitly via DOCKER_HOST (a \
+                 network-reachable daemon — a mounted unix socket is not reachable inside the \
+                 jail)."
+                    .to_string(),
+            ));
+        }
+        Some(start_per_job_podman(job_dir).await?)
+    } else {
+        None
+    };
+
+    // DOCKER_HOST handed to the script's docker CLI when using per-job podman: under
+    // nsjail the per-job socket is bind-mounted into the jail at /tmp/podman.sock
+    // (via {DOCKER_SOCK_MOUNT}); otherwise the script reaches the host socket directly.
+    let docker_host_for_script: Option<String> = {
+        #[cfg(feature = "dind")]
+        {
+            per_job_podman.as_ref().map(|p| {
+                if nsjail {
+                    "unix:///tmp/podman.sock".to_string()
+                } else {
+                    p.docker_host.clone()
+                }
+            })
+        }
+        #[cfg(not(feature = "dind"))]
+        {
+            None
+        }
+    };
+
+    // Forward DOCKER_HOST to the bash script in docker mode: the per-job podman
+    // socket when one was started, else the provided DOCKER_HOST / docker socket.
+    let docker_envs: Vec<(&str, String)> = if annotation.docker {
+        if let Some(dh) = &docker_host_for_script {
+            vec![("DOCKER_HOST", dh.clone())]
+        } else {
+            ["DOCKER_HOST", "DOCKER_TLS_VERIFY", "DOCKER_CERT_PATH"]
+                .iter()
+                .filter_map(|k| std::env::var(k).ok().map(|v| (*k, v)))
+                .collect()
+        }
+    } else {
+        vec![]
+    };
+
+    // nsjail mount block that exposes the per-job podman socket inside the jail.
+    let docker_sock_mount: String = {
+        #[cfg(feature = "dind")]
+        {
+            per_job_podman
+                .as_ref()
+                .map(|p| {
+                    format!(
+                        "mount {{\n    src: \"{}\"\n    dst: \"/tmp/podman.sock\"\n    is_bind: true\n    rw: true\n}}\n",
+                        p.host_sock
+                    )
+                })
+                .unwrap_or_default()
+        }
+        #[cfg(not(feature = "dind"))]
+        {
+            String::new()
+        }
+    };
+
     let child = if nsjail {
         let nsjail_timeout =
             resolve_nsjail_timeout(conn, &job.workspace_id, job.id, job.timeout).await;
@@ -219,6 +300,7 @@ exit $exit_status
                     "{TMP_MOUNT_BLOCK}",
                     &resolve_nsjail_tmp_mount_block(job_dir).await,
                 )
+                .replace("{DOCKER_SOCK_MOUNT}", &docker_sock_mount)
                 .replace("{TIMEOUT}", &nsjail_timeout),
         )?;
         let mut cmd_args = vec![
@@ -319,8 +401,11 @@ exit $exit_status
             worker_name,
             occupancy_metrics,
             _killpill_rx,
+            per_job_podman.as_ref().map(|p| p.docker_host.clone()),
         )
         .await;
+        // per_job_podman drops here (after handle_docker_job completes), tearing down
+        // the per-job podman instance so no container outlives the job.
     }
 
     let result_json_path = format!("{job_dir}/result.json");
@@ -369,8 +454,12 @@ async fn rm_container(client: &bollard::Docker, container_id: &str) {
 #[cfg(feature = "dind")]
 /// Connect to the Docker daemon, respecting DOCKER_HOST if set (e.g. for dind sidecar),
 /// otherwise falling back to the default unix socket at /var/run/docker.sock.
-fn connect_docker() -> Result<bollard::Docker, bollard::errors::Error> {
-    if std::env::var("DOCKER_HOST").is_ok() {
+fn connect_docker(docker_host: Option<&str>) -> Result<bollard::Docker, bollard::errors::Error> {
+    if let Some(dh) = docker_host {
+        // Per-job rootless podman: connect to the job's own unix socket.
+        let path = dh.strip_prefix("unix://").unwrap_or(dh);
+        bollard::Docker::connect_with_unix(path, 120, bollard::API_DEFAULT_VERSION)
+    } else if std::env::var("DOCKER_HOST").is_ok() {
         // DOCKER_HOST is set — use it (e.g. tcp://dind:2375 for docker-in-docker)
         bollard::Docker::connect_with_defaults()
     } else {
@@ -379,6 +468,180 @@ fn connect_docker() -> Result<bollard::Docker, bollard::errors::Error> {
     }
 }
 
+// A per-job rootless podman instance used for docker jobs (any sandbox mode —
+// nsjail, unshare, or none) when no Docker daemon is otherwise provided.
+// The job's docker CLI and handle_docker_job (in the worker) both talk to this
+// one instance's socket; because every container the job can create is registered
+// in this instance's isolated storage, tearing it down (`podman system reset`,
+// which also removes images and the subuid-owned overlay layers — see Drop)
+// guarantees no container outlives the job — even detached or extra ones.
+#[cfg(feature = "dind")]
+struct PerJobPodman {
+    dir: String,
+    host_sock: String,
+    docker_host: String,
+    service: Option<std::process::Child>,
+}
+
+// Blocking teardown of a per-job podman instance: `system reset` removes ALL
+// containers AND images in the per-job store and, crucially, deletes the
+// subuid-owned overlay layers a plain `rm -rf` (or `rm -af`, which only touches
+// containers) cannot — preventing a storage leak. Wrapped in `timeout` so a wedged
+// runtime/storage can't hang the caller indefinitely.
+#[cfg(feature = "dind")]
+fn teardown_per_job_podman(dir: String, service: Option<std::process::Child>) {
+    let storage = format!("{dir}/storage");
+    let runroot = format!("{dir}/runroot");
+    let _ = std::process::Command::new("timeout")
+        .args([
+            "30",
+            "podman",
+            "--root",
+            &storage,
+            "--runroot",
+            &runroot,
+            "system",
+            "reset",
+            "--force",
+        ])
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .status();
+    if let Some(mut child) = service {
+        // Kill the whole process group: when the worker is root the daemon runs as a
+        // `runuser` grandchild, so killing only the direct child would leak it.
+        unsafe { libc::kill(-(child.id() as i32), libc::SIGKILL) };
+        let _ = child.kill();
+        let _ = child.wait();
+    }
+    let _ = std::fs::remove_dir_all(&dir);
+}
+
+#[cfg(feature = "dind")]
+impl Drop for PerJobPodman {
+    fn drop(&mut self) {
+        let dir = std::mem::take(&mut self.dir);
+        let service = self.service.take();
+        // Teardown shells out (`podman system reset`) and can do non-trivial I/O, so
+        // offload it to the blocking pool rather than stalling the Tokio worker thread
+        // this Drop runs on. If we're not on a runtime (e.g. tests), run inline.
+        match tokio::runtime::Handle::try_current() {
+            Ok(handle) => {
+                handle.spawn_blocking(move || teardown_per_job_podman(dir, service));
+            }
+            Err(_) => teardown_per_job_podman(dir, service),
+        }
+    }
+}
+
+// Start a per-job rootless podman service in <job_dir>/podman with isolated
+// storage, returning the host socket path. Inherits the system container config
+// (/etc/containers) but overrides storage and the network backend via a per-job
+// user config so both are job-scoped (and rootful podman elsewhere is untouched).
+//
+// SECURITY: this is only ever started in NON-nsjail modes (unshare/none) — caller
+// refuses to auto-provide it under nsjail, since the daemon runs outside any sandbox.
+// A `# docker` script controls this daemon over the Docker API and can bind-mount any
+// WORLD-READABLE host path (e.g. `docker run -v /tmp/windmill/...` for other job dirs /
+// caches, or `-v /proc` for process cmdlines/args) — so it is NOT a filesystem sandbox,
+// and docker-capable workers stay a TRUSTED-TENANT capability (prefer dedicated/per-
+// workspace docker workers on shared fleets). What IS protected, when the worker runs
+// as root (the default) and the daemon is dropped to a non-root uid here: `0400` files
+// like the worker's `/proc/<worker>/environ` (DATABASE_URL etc.) stay root-owned and
+// unreadable by the uid-1000 container even via `-v /proc` or `--pid=host` (so don't
+// pass secrets as command-line args), and a container escape is unprivileged.
+#[cfg(feature = "dind")]
+async fn start_per_job_podman(job_dir: &str) -> Result<PerJobPodman, Error> {
+    use std::os::unix::process::CommandExt;
+    let dir = format!("{job_dir}/podman");
+    let xdg = format!("{dir}/xdg");
+    tokio::fs::create_dir_all(&xdg).await.map_err(to_anyhow)?;
+    // Default this rootless podman's container networking to slirp4netns: the
+    // netavark bridge default fails rootless on hosts without the required
+    // nftables setup. Scope it to THIS instance via a user (`$HOME`) config so we
+    // don't touch the global `/etc/containers` config (which would also change
+    // rootful podman behavior for anyone else in the image). `$HOME` is the job
+    // dir below for both the root-drop and non-root launch paths.
+    let containers_conf_dir = format!("{dir}/.config/containers");
+    tokio::fs::create_dir_all(&containers_conf_dir)
+        .await
+        .map_err(to_anyhow)?;
+    tokio::fs::write(
+        format!("{containers_conf_dir}/containers.conf"),
+        "[containers]\nnetns = \"slirp4netns\"\n",
+    )
+    .await
+    .map_err(to_anyhow)?;
+    let host_sock = format!("{dir}/podman.sock");
+    let _ = tokio::fs::remove_file(&host_sock).await;
+    let storage = format!("{dir}/storage");
+    let runroot = format!("{dir}/runroot");
+    let sock_url = format!("unix://{host_sock}");
+
+    // If the worker is root (the default for docker workers), drop the per-job podman
+    // to a non-root uid via `runuser` so it runs ROOTLESS. This keeps the worker's
+    // secrets in `/proc/<worker>/environ` (DATABASE_URL etc.) root-owned and therefore
+    // unreadable by the uid-1000 container even via `docker run --pid=host`, and makes
+    // a container escape land as an unprivileged user. NB: it does NOT confine the
+    // daemon's filesystem view — a script can still `docker run -v <worker path>` to
+    // reach other job dirs / caches, so docker-capable workers remain trusted-tenant.
+    // A non-root worker can't drop further (it runs podman rootless as itself, and a
+    // `--pid=host` container shares its uid) — run the docker worker as root for the
+    // /proc env protection. `process_group(0)` lets teardown kill the whole tree (with
+    // `runuser`, podman is a grandchild, not the direct child).
+    let mut cmd = if unsafe { libc::geteuid() } == 0 {
+        let launch = format!(
+            "set -e\nchown -R 1000:1000 '{dir}'\nexec runuser -u \"$(id -nu 1000)\" -- env \
+             HOME='{dir}' XDG_RUNTIME_DIR='{xdg}' podman --root '{storage}' --runroot '{runroot}' \
+             system service --time=0 '{sock_url}'\n"
+        );
+        let mut c = std::process::Command::new("bash");
+        c.args(["-c", &launch]);
+        c
+    } else {
+        let mut c = std::process::Command::new("podman");
+        // HOME=<job dir> so podman picks up the per-job containers.conf written
+        // above (netns override) and keeps its config/cache job-scoped.
+        c.env("HOME", &dir);
+        c.args([
+            "--root",
+            &storage,
+            "--runroot",
+            &runroot,
+            "system",
+            "service",
+            "--time=0",
+            &sock_url,
+        ]);
+        c
+    };
+    let service = cmd
+        .process_group(0)
+        .stdout(std::process::Stdio::null())
+        .stderr(std::process::Stdio::null())
+        .spawn()
+        .map_err(|e| {
+            Error::ExecutionErr(format!(
+                "no Docker daemon provided (DOCKER_HOST/socket) and failed to start the per-job \
+                 podman runtime: {e}. Use a windmill *-full image (ships podman), or provide a \
+                 Docker daemon via DOCKER_HOST or a mounted /var/run/docker.sock."
+            ))
+        })?;
+    // Wait (up to ~10s) for the rootless podman service socket to appear.
+    for _ in 0..50 {
+        if tokio::fs::metadata(&host_sock).await.is_ok() {
+            break;
+        }
+        tokio::time::sleep(std::time::Duration::from_millis(200)).await;
+    }
+    Ok(PerJobPodman {
+        dir,
+        docker_host: format!("unix://{host_sock}"),
+        host_sock,
+        service: Some(service),
+    })
+}
+
 #[cfg(feature = "dind")]
 async fn handle_docker_job(
     job_id: Uuid,
@@ -390,10 +653,13 @@ async fn handle_docker_job(
     worker_name: &str,
     occupancy_metrics: &mut OccupancyMetrics,
     killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
+    // Some(socket) when a per-job rootless podman runs the container;
+    // None to use the provided DOCKER_HOST / default socket (legacy).
+    docker_host: Option<String>,
 ) -> Result<Box<RawValue>, Error> {
     use crate::job_logger::append_logs_with_compaction;
 
-    let client = connect_docker().map_err(to_anyhow)?;
+    let client = connect_docker(docker_host.as_deref()).map_err(to_anyhow)?;
 
     let container_id = job_id.to_string();
     let inspected = client.inspect_container(&container_id, None).await;
@@ -437,8 +703,9 @@ async fn handle_docker_job(
     let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
     let workspace_id2 = workspace_id.to_string();
     let mut killpill_rx = killpill_rx.resubscribe();
+    let docker_host_logs = docker_host.clone();
     let logs = tokio::spawn(async move {
-        let client = connect_docker().map_err(to_anyhow);
+        let client = connect_docker(docker_host_logs.as_deref()).map_err(to_anyhow);
         if let Ok(client) = client {
             let mut log_stream = client.logs(
                 &ncontainer_id,
@@ -506,7 +773,7 @@ async fn handle_docker_job(
         }
     });
 
-    let mem_client = connect_docker().map_err(to_anyhow);
+    let mem_client = connect_docker(docker_host.as_deref()).map_err(to_anyhow);
     let ncontainer_id = container_id.clone();
     let result = run_future_with_polling_update_job_poller(
         job_id,

docker-compose.yml

@@ -49,27 +49,40 @@ services:
 
     logging: *default-logging
 
-  # Docker-in-Docker sidecar: provides an isolated Docker daemon so user scripts
-  # can run containers without accessing the host Docker socket.
-  dind:
-    image: docker:dind
-    privileged: true
-    restart: unless-stopped
-    environment:
-      DOCKER_TLS_CERTDIR: ""
-    volumes:
-      - dind-data:/var/lib/docker
-    expose:
-      - 2375
-    healthcheck:
-      test: ["CMD", "docker", "info"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    logging: *default-logging
-
   windmill_worker:
     image: ${WM_IMAGE}
+    # --- To run `# docker` scripts (bash scripts with a `# docker` annotation) on
+    #     this worker: comment the `image` line above and uncomment the *-full
+    #     image below. Each docker job then runs in its OWN ephemeral rootless
+    #     podman, torn down with the job — no dind sidecar, no host Docker socket,
+    #     your scripts unchanged. The *-full image also bundles the heavier runtimes
+    #     (Java, .NET, Ruby, R, Rust, Ansible, Nushell). `# docker` scripts are
+    #     auto-tagged `docker` (served by default workers); to send them to a
+    #     dedicated/bigger group instead, run a worker with WORKER_TAGS=docker. ---
+    # image: ghcr.io/windmill-labs/windmill-full:main   # windmill-ee-full:main for EE
+    # On old kernels (<5.13, no native rootless overlay) podman needs fuse-overlayfs;
+    # expose the device then (harmless to keep; auto-provided by `privileged` if the
+    # host has it):
+    # devices:
+    #   - /dev/fuse
+    # Keep this worker running as ROOT (the default — do NOT set `user: 1000`):
+    # windmill itself drops the per-job podman to a non-root uid, so containers are
+    # rootless AND the worker's own secrets in /proc (DATABASE_URL etc.) stay
+    # root-owned, unreadable by a `docker run --pid=host` container. (NB: this does
+    # not confine the container's filesystem view — a `# docker` script can still
+    # `docker run -v` to read WORLD-READABLE host state: other job dirs' world-readable
+    # files, the dep cache, and world-readable /proc like process cmdlines/args — though
+    # NOT 0400 files like /proc/<pid>/environ (env secrets stay safe; don't pass secrets
+    # as args). So treat docker workers as trusted-tenant / dedicate them on shared
+    # fleets.) Docker jobs use disk like any other job (no docker-specific cap); bound
+    # it at the infra level (e.g. a sized volume for /tmp/windmill) for all jobs.
+    # If you enable nsjail (DISABLE_NSJAIL=false or job isolation = nsjail), per-job
+    # podman is NOT auto-provided for docker jobs — nsjail signals "fully sandbox jobs"
+    # and the podman daemon runs outside the jail, so docker jobs then require an
+    # explicit DOCKER_HOST or a separate non-nsjail worker group. (nsjail is off by
+    # default; the default worker uses unshare, so docker works out of the box here.)
+    # To use an external/host Docker daemon instead (legacy): set DOCKER_HOST or mount
+    # /var/run/docker.sock.
     pull_policy: always
     deploy:
       replicas: 3
@@ -86,26 +99,14 @@ services:
       - DATABASE_URL=${DATABASE_URL}
       - MODE=worker
       - WORKER_GROUP=default
-      # If running with non-root/non-windmill UID (e.g., user: "1001:1001"),
-      # add: - HOME=/tmp
       - FAVOR_UNSHARE_PID=true
-      # Connect to the dind sidecar instead of the host Docker socket
-      - DOCKER_HOST=tcp://dind:2375
     depends_on:
       db:
         condition: service_healthy
-      dind:
-        condition: service_healthy
     # to mount the worker folder to debug, KEEP_JOB_DIR=true and mount /tmp/windmill
     volumes:
       - worker_dependency_cache:/tmp/windmill/cache
       - worker_logs:/tmp/windmill/logs
-      ## WARNING: mounting the host Docker socket grants user scripts full access to
-      ## the host Docker daemon, enabling host filesystem access and privilege escalation.
-      ## Only use this if you fully trust all users who can run scripts.
-      ## To use it, remove the DOCKER_HOST env var and dind depends_on above,
-      ## and uncomment the line below:
-      # - /var/run/docker.sock:/var/run/docker.sock
 
     logging: *default-logging
 
@@ -237,4 +238,3 @@ volumes:
   windmill_index: null
   lsp_cache: null
   caddy_data: null
-  dind-data: null

docker/DockerfileFull

@@ -31,6 +31,35 @@ RUN apt-get install -y ruby ruby-bundler
 RUN apt-get install -y r-base-dev \
     && Rscript -e 'install.packages("renv", lib="/usr/lib/R/library", repos="https://cloud.r-project.org")'
 
+# Rootless container runtime (podman) for docker-mode jobs. When a `# docker` job
+# runs on a worker with no Docker daemon provided (no DOCKER_HOST, no mounted
+# /var/run/docker.sock), Windmill automatically starts a per-job rootless podman —
+# no privileged dind sidecar or host Docker socket. Run the docker worker group as
+# `user: "1000:1000"` for a rootless (unprivileged) daemon; running as root still
+# works but is rootful (less isolated).
+RUN apt-get -y update \
+    && apt-get install -y --no-install-recommends \
+        podman \
+        uidmap \
+        fuse-overlayfs \
+        slirp4netns \
+        crun \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+RUN useradd -u 1000 -m -s /bin/bash windmill 2>/dev/null || true
+# Ensure a subuid/subgid range exists for rootless podman (useradd usually adds
+# one already; only append if it didn't, to avoid a duplicate range).
+RUN grep -q '^windmill:' /etc/subuid || echo "windmill:100000:65536" >> /etc/subuid; \
+    grep -q '^windmill:' /etc/subgid || echo "windmill:100000:65536" >> /etc/subgid
+# newuidmap/newgidmap need privilege to map the subuid range for rootless podman;
+# ensure they are setuid in case the package's file capabilities are lost in the
+# image layers.
+RUN chmod u+s /usr/bin/newuidmap /usr/bin/newgidmap
+# NB: rootless container networking is defaulted to slirp4netns (the netavark
+# bridge default fails rootless without nftables setup) per-job by the worker via
+# a job-scoped containers.conf, rather than globally here, so rootful podman is
+# unaffected.
+
 # Fix UV cache permissions for non-root user support (uid 1000, etc.)
 # The uv tool install ansible command populates the UV cache with root-owned files
 RUN chmod -R a+rw /tmp/windmill/cache/uv && \

docker/DockerfileFullEe

@@ -61,6 +61,35 @@ RUN apt-get install -y iptables
 # Kerberos runtime
 RUN apt-get install -y libsasl2-modules-gssapi-mit krb5-user
 
+# Rootless container runtime (podman) for docker-mode jobs. When a `# docker` job
+# runs on a worker with no Docker daemon provided (no DOCKER_HOST, no mounted
+# /var/run/docker.sock), Windmill automatically starts a per-job rootless podman —
+# no privileged dind sidecar or host Docker socket. Run the docker worker group as
+# `user: "1000:1000"` for a rootless (unprivileged) daemon; running as root still
+# works but is rootful (less isolated).
+RUN apt-get -y update \
+    && apt-get install -y --no-install-recommends \
+        podman \
+        uidmap \
+        fuse-overlayfs \
+        slirp4netns \
+        crun \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+RUN useradd -u 1000 -m -s /bin/bash windmill 2>/dev/null || true
+# Ensure a subuid/subgid range exists for rootless podman (useradd usually adds
+# one already; only append if it didn't, to avoid a duplicate range).
+RUN grep -q '^windmill:' /etc/subuid || echo "windmill:100000:65536" >> /etc/subuid; \
+    grep -q '^windmill:' /etc/subgid || echo "windmill:100000:65536" >> /etc/subgid
+# newuidmap/newgidmap need privilege to map the subuid range for rootless podman;
+# ensure they are setuid in case the package's file capabilities are lost in the
+# image layers.
+RUN chmod u+s /usr/bin/newuidmap /usr/bin/newgidmap
+# NB: rootless container networking is defaulted to slirp4netns (the netavark
+# bridge default fails rootless without nftables setup) per-job by the worker via
+# a job-scoped containers.conf, rather than globally here, so rootful podman is
+# unaffected.
+
 # Fix UV cache permissions for non-root user support (uid 1000, etc.)
 # The uv tool install ansible command populates the UV cache with root-owned files
 RUN chmod -R a+rw /tmp/windmill/cache/uv && \

frontend/src/lib/components/ScriptBuilder.svelte

@@ -1007,21 +1007,6 @@
 
 	function onScriptLanguageTrigger(lang: 'docker' | 'bunnative' | ScriptLang) {
 		if (lang == 'docker') {
-			if (isCloudHosted()) {
-				sendUserToast(
-					'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.',
-					true,
-					[
-						{
-							label: 'Learn more',
-							callback: () => {
-								window.open('https://www.windmill.dev/docs/advanced/docker', '_blank')
-							}
-						}
-					]
-				)
-				return
-			}
 			template = 'docker'
 		} else if (lang == 'bunnative') {
 			template = 'bunnative'

frontend/src/lib/components/flows/content/FlowInputs.svelte

@@ -7,8 +7,6 @@
 	import FlowScriptPicker from '../pickers/FlowScriptPicker.svelte'
 	import PickHubScript from '../pickers/PickHubScript.svelte'
 	import WorkspaceScriptPicker from '../pickers/WorkspaceScriptPicker.svelte'
-	import { isCloudHosted } from '$lib/cloud'
-	import { sendUserToast } from '$lib/toast'
 	import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
 	import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
 	import { Check, Code, Zap } from 'lucide-svelte'
@@ -259,23 +257,6 @@
 						{label}
 						lang={lang == 'docker' ? 'bash' : lang}
 						on:click={() => {
-							if (lang == 'docker') {
-								if (isCloudHosted()) {
-									sendUserToast(
-										'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.',
-										true,
-										[
-											{
-												label: 'Learn more',
-												callback: () => {
-													window.open('https://www.windmill.dev/docs/advanced/docker', '_blank')
-												}
-											}
-										]
-									)
-									return
-								}
-							}
 							dispatch('new', {
 								language: lang == 'docker' ? 'bash' : lang,
 								kind,

frontend/src/lib/components/flows/content/FlowInputsQuick.svelte

@@ -4,7 +4,6 @@
 </script>
 
 <script lang="ts">
-	import { isCloudHosted } from '$lib/cloud'
 	import { sendUserToast } from '$lib/toast'
 	import FlowScriptPickerQuick from '../pickers/FlowScriptPickerQuick.svelte'
 	import { defaultScriptLanguages, processLangs } from '$lib/scripts'
@@ -401,24 +400,6 @@
 					{label}
 					lang={lang == 'docker' ? 'bash' : lang}
 					on:click={() => {
-						if (lang == 'docker') {
-							if (isCloudHosted()) {
-								sendUserToast(
-									'You cannot use Docker scripts on the multi-tenant platform. Use a dedicated instance or self-host windmill instead.',
-									true,
-									[
-										{
-											label: 'Learn more',
-											callback: () => {
-												window.open('https://www.windmill.dev/docs/advanced/docker', '_blank')
-											}
-										}
-									]
-								)
-								return
-							}
-						}
-
 						dispatch('new', {
 							kind: selectedKind,
 							inlineScript: {