diff --git a/README.md b/README.md index 90c6b90ce3..b8e37471bd 100644 --- a/README.md +++ b/README.md @@ -208,15 +208,18 @@ docker compose up -d Go to http://localhost - default credentials: `admin@windmill.dev` / `changeme` > [!NOTE] -> To run `# docker` scripts (bash scripts with the `# docker` annotation): these run -> on the default worker like any bash job. Just give that worker a `*-full` image -> (ships podman) and the `/dev/fuse` device. On a worker with **no Docker daemon -> provided** (no `DOCKER_HOST`, no mounted `/var/run/docker.sock`), Windmill then runs -> each docker job in its own ephemeral **rootless podman**, torn down with the job — no -> privileged daemon, no host socket, scripts unchanged. The `*-full` image is also the -> batteries-included runtime — on top of the base (TS/Bun/Deno, Python, Go) it adds -> Java, .NET, Ruby, R, Rust, Ansible, and Nushell (plus Oracle/Kerberos in EE). See the -> default `windmill_worker` notes in +> To run `# docker` scripts (bash scripts with the `# docker` annotation): just give a +> worker a `*-full` image (ships podman). `# docker` scripts are auto-tagged `docker` +> (served by default workers out of the box; route them to a dedicated/bigger group +> with `WORKER_TAGS=docker`). On a worker with **no Docker daemon provided** (no +> `DOCKER_HOST`, no mounted `/var/run/docker.sock`), Windmill runs each docker job in +> its own ephemeral **rootless podman**, torn down with the job — no privileged daemon, +> no host socket, scripts unchanged. Per-job image storage is capped via the +> `docker_image_storage_size_mb` instance setting (default 8GB). On old kernels (<5.13) +> also expose `/dev/fuse`. The `*-full` image is also the batteries-included runtime — +> on top of the base (TS/Bun/Deno, Python, Go) it adds Java, .NET, Ruby, R, Rust, +> Ansible, and Nushell (plus Oracle/Kerberos in EE). See the default `windmill_worker` +> notes in > [docker-compose.yml](./docker-compose.yml). To use an external/host Docker daemon > instead (legacy), provide `DOCKER_HOST` or mount `/var/run/docker.sock`. diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 13f52037e7..07e5a8a3c9 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -57,20 +57,21 @@ use windmill_common::{ CRITICAL_ALERTS_ON_TOKEN_EXPIRY_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_PASSWORD_LOGIN, DISABLE_PASSWORD_LOGIN_SETTING, - EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, - FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, HUB_API_SECRET_SETTING, - HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, - JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, - MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, - NSJAIL_TMPFS_SIZE_MB_SETTING, NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, - OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, PIP_INDEX_URL_SETTING, - POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, PREVIEW_TAGS_OVERRIDE_SETTING, - REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, - RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, - STORE_AUDIT_LOGS_S3_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, - UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, - WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, - WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, + DOCKER_IMAGE_STORAGE_SIZE_MB_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING, + EXTRA_PIP_INDEX_URL_SETTING, FORK_WORKSPACE_TAG_APPEND_FORK_SUFFIX_SETTING, + HUB_API_SECRET_SETTING, HUB_BASE_URL_SETTING, INSTANCE_PYTHON_VERSION_SETTING, + JOB_DEFAULT_TIMEOUT_SECS_SETTING, JOB_ISOLATION_SETTING, JWT_SECRET_SETTING, + KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, + NPMRC_SETTING, NPM_CONFIG_REGISTRY_SETTING, NSJAIL_TMPFS_SIZE_MB_SETTING, + NSJAIL_TMP_BACKING_SETTING, NUGET_CONFIG_SETTING, OTEL_SETTING, OTEL_TRACING_PROXY_SETTING, + PIP_INDEX_URL_SETTING, POWERSHELL_REPO_PAT_SETTING, POWERSHELL_REPO_URL_SETTING, + PREVIEW_TAGS_OVERRIDE_SETTING, REQUEST_SIZE_LIMIT_SETTING, + REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, STORE_AUDIT_LOGS_S3_SETTING, + TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, + UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, + WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, + WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, }, indexer::load_indexer_config, jwt::JWT_SECRET, @@ -108,12 +109,13 @@ use windmill_queue::{cancel_job, get_queued_job_v2, SameWorkerPayload}; use windmill_worker::{ result_processor::handle_job_error, JobCompletedSender, JobIsolationLevel, OtelTracingProxySettings, SameWorkerSender, WorkspaceRegistryMap, BUNFIG_INSTALL_SCOPES, - BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, - JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, MAVEN_REPOS, MAVEN_SETTINGS_XML, - NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, - NSJAIL_TMP_BACKING, NUGET_CONFIG, OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, POWERSHELL_REPO_PAT, POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, - UV_INDEX_STRATEGY, UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, + BUN_INSTALL_MIN_RELEASE_AGE, CARGO_REGISTRIES, DOCKER_IMAGE_STORAGE_SIZE_MB, + INSTANCE_PYTHON_VERSION, JAVA_HOME_DIR, JOB_DEFAULT_TIMEOUT, JOB_ISOLATION, KEEP_JOB_DIR, + MAVEN_REPOS, MAVEN_SETTINGS_XML, NO_DEFAULT_MAVEN, NPMRC, NPM_CONFIG_REGISTRY, + NSJAIL_AVAILABLE, NSJAIL_TMPFS_SIZE_MB, NSJAIL_TMP_BACKING, NUGET_CONFIG, + OTEL_TRACING_PROXY_SETTINGS, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, POWERSHELL_REPO_PAT, + POWERSHELL_REPO_URL, UNSHARE_PATH, UV_EXCLUDE_NEWER, UV_INDEX_STRATEGY, + UV_PYTHON_INSTALL_MIRROR, WORKSPACE_REGISTRIES, }; #[cfg(feature = "parquet")] @@ -406,6 +408,7 @@ pub async fn initial_load( reload_job_default_timeout_setting(&conn).await; reload_job_isolation_setting(&conn).await; reload_nsjail_tmpfs_size_setting(&conn).await; + reload_docker_image_storage_size_setting(&conn).await; reload_nsjail_tmp_backing_setting(&conn).await; reload_extra_pip_index_url_setting(&conn).await; reload_pip_index_url_setting(&conn).await; @@ -2035,6 +2038,16 @@ pub async fn reload_nsjail_tmpfs_size_setting(conn: &Connection) { .await; } +pub async fn reload_docker_image_storage_size_setting(conn: &Connection) { + reload_option_setting_with_tracing( + conn, + DOCKER_IMAGE_STORAGE_SIZE_MB_SETTING, + "DOCKER_IMAGE_STORAGE_SIZE_MB", + DOCKER_IMAGE_STORAGE_SIZE_MB.clone(), + ) + .await; +} + pub async fn reload_nsjail_tmp_backing_setting(conn: &Connection) { reload_option_setting_with_tracing( conn, diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index 0dc86bfda8..8fbc9c1407 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -894,6 +894,19 @@ async fn create_script_internal<'c>( } check_scopes(&authed, || format!("scripts:write:{}", ns.path))?; + // Auto-route Bash `# docker` scripts to the `docker` tag (part of DEFAULT_TAGS) + // when no explicit tag is set, so they can be directed to docker-capable / bigger + // workers. Done here — before no-op detection and the insert — so an unchanged + // redeploy is still correctly detected as a no-op. Tag-only mechanism mirroring + // the routing half of bunnative/nativets (the bash executor handles `# docker` + // at runtime, so no distinct ScriptLang is needed). + if ns.tag.as_deref().map_or(true, |t| t.is_empty()) + && ns.language == ScriptLang::Bash + && windmill_common::worker::BashAnnotations::parse(&ns.content).docker + { + ns.tag = Some(windmill_common::worker::DOCKER_BASH_TAG.to_string()); + } + guard_script_from_debounce_data(&ns).await?; let codebase = ns.codebase.as_ref(); diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 75f3fdf06b..12d06e7de3 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -58,6 +58,11 @@ pub const NSJAIL_TMPFS_SIZE_MB_SETTING: &str = "nsjail_tmpfs_size_mb"; pub const NSJAIL_TMP_BACKING_SETTING: &str = "nsjail_tmp_backing"; pub const NSJAIL_TMP_BACKING_DISK: &str = "disk"; pub const NSJAIL_TMP_BACKING_TMPFS: &str = "tmpfs"; +// Soft size cap (MB) for the per-job rootless-podman image store of `# docker` +// jobs, enforced by polling (a uid-1000 rootless worker cannot mount a sized +// tmpfs): past the cap the worker logs an error and tears the runtime down, +// failing the job. Empty -> a generous default; 0 -> uncapped (host-disk bounded). +pub const DOCKER_IMAGE_STORAGE_SIZE_MB_SETTING: &str = "docker_image_storage_size_mb"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 92ebf08477..e788fcdb29 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -169,6 +169,10 @@ lazy_static::lazy_static! { "python3".to_string(), "go".to_string(), "bash".to_string(), + // Bash `# docker` scripts are auto-tagged `docker` (see DOCKER_BASH_TAG / + // BashAnnotations) so they can be routed to docker-capable workers; in + // DEFAULT_TAGS so default workers serve them out of the box. + DOCKER_BASH_TAG.to_string(), "powershell".to_string(), "nativets".to_string(), "mysql".to_string(), @@ -859,6 +863,15 @@ pub struct BashAnnotations { pub sandbox: bool, } +/// Tag assigned to Bash scripts carrying the `# docker` annotation so docker +/// jobs can be routed to docker-capable / bigger workers. This is a tag-only +/// mechanism (no distinct ScriptLang — the bash executor handles the annotation +/// at runtime), mirroring the routing half of `bunnative`/`nativets`. It is part +/// of DEFAULT_TAGS so default workers serve it out of the box; derived at script +/// create (stored on `script.tag`) and at preview push when no explicit tag is +/// set. Callers gate on `language == Bash`. +pub const DOCKER_BASH_TAG: &str = "docker"; + #[derive(Debug, Clone, Copy, PartialEq)] pub enum SqlResultCollectionStrategy { LastStatementAllRows, diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 7c4a832760..c8444217c5 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -5744,6 +5744,14 @@ async fn push_inner<'c, 'd>( } else { ScriptLang::Nativets.as_str() } + } else if x == &ScriptLang::Bash + && raw_code.as_deref().is_some_and(|c| { + windmill_common::worker::BashAnnotations::parse(c).docker + }) + { + // Bash `# docker` previews/raw runs route to the `docker` + // tag (deployed scripts get it stored at create time). + windmill_common::worker::DOCKER_BASH_TAG } else { x.as_str() }; diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 9425a969a6..117d31cc15 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -205,7 +205,7 @@ exit $exit_status || std::path::Path::new("/var/run/docker.sock").exists(); #[cfg(feature = "dind")] let per_job_podman: Option = if annotation.docker && !docker_daemon_provided { - Some(start_per_job_podman(job_dir).await?) + Some(start_per_job_podman(job_dir, job.id, &job.workspace_id, conn).await?) } else { None }; @@ -462,11 +462,17 @@ struct PerJobPodman { host_sock: String, docker_host: String, service: Option, + // Background task enforcing the soft image-store size cap; aborted on Drop. + monitor: Option>, } #[cfg(feature = "dind")] impl Drop for PerJobPodman { fn drop(&mut self) { + // Stop the soft size-cap monitor first so it can't fire mid-teardown. + if let Some(monitor) = self.monitor.take() { + monitor.abort(); + } // Drop is synchronous: use blocking std::process for guaranteed teardown. // `system reset` removes ALL containers AND images in the per-job store and, // crucially, deletes the subuid-owned overlay layers a plain `rm -rf` (or @@ -494,11 +500,125 @@ impl Drop for PerJobPodman { } } +// Default soft cap (MB) for the per-job podman image store when +// `docker_image_storage_size_mb` is unset. Images are large, so this is generous; +// an explicit value of 0 (or negative) disables the cap entirely. +#[cfg(feature = "dind")] +const DEFAULT_DOCKER_IMAGE_STORAGE_SIZE_MB: i64 = 8192; + +// Resolve the soft size cap (bytes) for the per-job image store from the +// `docker_image_storage_size_mb` instance setting. `None` means uncapped. +#[cfg(feature = "dind")] +async fn docker_image_store_limit_bytes() -> Option { + let mb = match *crate::worker::DOCKER_IMAGE_STORAGE_SIZE_MB.read().await { + Some(mb) if mb <= 0 => return None, // explicitly disabled + Some(mb) => mb, + None => DEFAULT_DOCKER_IMAGE_STORAGE_SIZE_MB, + }; + Some(mb as u64 * 1024 * 1024) +} + +// Measure the on-disk size (bytes) of the per-job podman graphroot. The overlay +// layers are owned by remapped subuids, so a plain `du` as the worker user cannot +// traverse them — run `du` inside the podman user namespace (`podman unshare`) +// where those files map back to the caller. +#[cfg(feature = "dind")] +async fn podman_store_size_bytes(root: &str, runroot: &str) -> Option { + let out = tokio::process::Command::new("podman") + .args([ + "--root", + root, + "--runroot", + runroot, + "unshare", + "du", + "-sb", + root, + ]) + .output() + .await + .ok()?; + std::str::from_utf8(&out.stdout) + .ok()? + .split_whitespace() + .next()? + .parse::() + .ok() +} + +// Soft size cap for the per-job image store, enforced by polling (the +// rootless-compatible alternative to a sized tmpfs — a uid-1000 worker cannot +// mount one). If the graphroot exceeds the limit, log a clear error and tear the +// runtime down: kill the service so an in-flight `docker pull` fails, and +// `system reset` to stop any running container and free the space. Soft: a job can +// overshoot by up to one poll interval of writes before being caught. +#[cfg(feature = "dind")] +fn spawn_docker_storage_monitor( + root: String, + runroot: String, + service_pid: Option, + limit_bytes: u64, + job_id: Uuid, + workspace_id: String, + conn: Connection, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + loop { + tokio::time::sleep(std::time::Duration::from_secs(2)).await; + let Some(size) = podman_store_size_bytes(&root, &runroot).await else { + continue; + }; + if size > limit_bytes { + let limit_mb = limit_bytes / (1024 * 1024); + let used_mb = size / (1024 * 1024); + append_logs( + &job_id, + &workspace_id, + &format!( + "\nERROR: docker image storage for this job reached ~{used_mb}MB, over \ + the {limit_mb}MB limit (docker_image_storage_size_mb). Aborting the \ + docker runtime — raise the limit or route this job to a worker with \ + more disk.\n" + ), + &conn, + ) + .await; + if let Some(pid) = service_pid { + let _ = std::process::Command::new("kill") + .arg("-9") + .arg(pid.to_string()) + .status(); + } + let _ = std::process::Command::new("podman") + .args([ + "--root", + &root, + "--runroot", + &runroot, + "system", + "reset", + "--force", + ]) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status(); + break; + } + } + }) +} + // Start a per-job rootless podman service in /podman with isolated // storage, returning the host socket path. Inherits the worker's container -// config (CONTAINERS_CONF etc.) but overrides storage so it is job-scoped. +// config (CONTAINERS_CONF etc.) but overrides storage so it is job-scoped. A +// background monitor enforces a soft size cap on the image store. #[cfg(feature = "dind")] -async fn start_per_job_podman(job_dir: &str) -> Result { +async fn start_per_job_podman( + job_dir: &str, + job_id: Uuid, + workspace_id: &str, + conn: &Connection, +) -> Result { let dir = format!("{job_dir}/podman"); tokio::fs::create_dir_all(&dir).await.map_err(to_anyhow)?; let host_sock = format!("{dir}/podman.sock"); @@ -526,6 +646,7 @@ async fn start_per_job_podman(job_dir: &str) -> Result { Docker daemon via DOCKER_HOST or a mounted /var/run/docker.sock." )) })?; + let service_pid = service.id(); // Wait (up to ~10s) for the rootless podman service socket to appear. for _ in 0..50 { if tokio::fs::metadata(&host_sock).await.is_ok() { @@ -533,11 +654,24 @@ async fn start_per_job_podman(job_dir: &str) -> Result { } tokio::time::sleep(std::time::Duration::from_millis(200)).await; } + // Soft size-cap monitor for the image store (rootless-compatible enforcement). + let monitor = docker_image_store_limit_bytes().await.map(|limit_bytes| { + spawn_docker_storage_monitor( + storage.clone(), + runroot.clone(), + Some(service_pid), + limit_bytes, + job_id, + workspace_id.to_string(), + conn.clone(), + ) + }); Ok(PerJobPodman { dir, docker_host: format!("unix://{host_sock}"), host_sock, service: Some(service), + monitor, }) } diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 7fb0dee9f5..063268ab49 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -694,6 +694,14 @@ lazy_static::lazy_static! { /// RAM-backed tmpfs sized by `nsjail_tmpfs_size_mb`. pub static ref NSJAIL_TMP_BACKING: Arc>> = Arc::new(RwLock::new(None)); + /// Soft size cap (MB) for the per-job rootless-podman image store of `# docker` + /// jobs. Enforced by polling (a uid-1000 rootless worker cannot mount a sized + /// tmpfs): if the store exceeds the cap the worker logs a clear error and tears + /// the per-job runtime down, failing the job. When `None`, falls back to + /// `DEFAULT_DOCKER_IMAGE_STORAGE_SIZE_MB`; an explicit `0`/negative disables the + /// cap (uncapped, bounded only by host disk). + pub static ref DOCKER_IMAGE_STORAGE_SIZE_MB: Arc>> = Arc::new(RwLock::new(None)); + /// Optional mirror URL for `uv python install`. Wires to the `UV_PYTHON_INSTALL_MIRROR` /// env var when forwarded to uv. Can be set via the `UV_PYTHON_INSTALL_MIRROR` env var /// or the `uv_python_install_mirror` instance setting. diff --git a/docker-compose.yml b/docker-compose.yml index 5acd1c8867..ec426db8c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -51,6 +51,27 @@ services: windmill_worker: image: ${WM_IMAGE} + # --- To run `# docker` scripts (bash scripts with a `# docker` annotation) on + # this worker: comment the `image` line above and uncomment the *-full + # image below. Each docker job then runs in its OWN ephemeral rootless + # podman, torn down with the job — no dind sidecar, no host Docker socket, + # your scripts unchanged. The *-full image also bundles the heavier runtimes + # (Java, .NET, Ruby, R, Rust, Ansible, Nushell). `# docker` scripts are + # auto-tagged `docker` (served by default workers); to send them to a + # dedicated/bigger group instead, run a worker with WORKER_TAGS=docker. --- + # image: ghcr.io/windmill-labs/windmill-full:main # windmill-ee-full:main for EE + # On old kernels (<5.13, no native rootless overlay) podman needs fuse-overlayfs; + # expose the device then (harmless to keep; auto-provided by `privileged` if the + # host has it): + # devices: + # - /dev/fuse + # Optional hardening: run rootless (a container escape lands as an unprivileged + # user) by uncommenting `user` below and `- HOME=/tmp` under environment — the + # named cache volumes must then be writable by uid 1000. To use an external/host + # Docker daemon instead (legacy): set DOCKER_HOST or mount /var/run/docker.sock. + # Cap per-job image storage via the `docker_image_storage_size_mb` instance + # setting (default 8GB; the worker aborts a job whose images exceed it). + # user: "1000:1000" pull_policy: always deploy: replicas: 3 @@ -67,8 +88,7 @@ services: - DATABASE_URL=${DATABASE_URL} - MODE=worker - WORKER_GROUP=default - # If running with non-root/non-windmill UID (e.g., user: "1001:1001"), - # add: - HOME=/tmp + # - HOME=/tmp # required when running as non-root (see `user` above) - FAVOR_UNSHARE_PID=true depends_on: db: @@ -80,33 +100,6 @@ services: logging: *default-logging - ## Running `# docker` scripts (bash scripts with the `# docker` annotation that - ## do `docker pull` / `docker run --name $WM_JOB_ID ...`): these run on the - ## default worker like any bash job — no dedicated worker group or custom tag. - ## To enable them, give the worker above two things: - ## 1. a *-full image (ships podman + the docker CLI): - ## image: ghcr.io/windmill-labs/windmill-full:main # windmill-ee-full:main for EE - ## 2. the /dev/fuse device (rootless fuse-overlayfs storage): - ## devices: - ## - /dev/fuse - ## Each docker job then spins up its OWN ephemeral rootless podman, torn down - ## with the job — no privileged dind sidecar and no host Docker socket. Your - ## scripts are unchanged. - ## - ## The *-full image is also the batteries-included runtime: on top of the base - ## (TypeScript/Bun/Deno, Python, Go, bash) it adds Java, .NET (C#), Ruby, R, - ## Rust, Ansible, and Nushell — plus the Oracle DB client and Kerberos in EE — - ## so the same worker can run those jobs too. - ## - ## Hardening: the worker above runs as root (rootful podman, still works). For - ## the rootless benefit (a container escape lands as an unprivileged user) add - ## `user: "1000:1000"` and `- HOME=/tmp`. Full memory monitoring (mem_peak) - ## needs cgroup v2 delegation from the host. For the strongest isolation, run - ## docker workloads on a dedicated host/node. - ## - ## Legacy: to use an external/host Docker daemon instead of the bundled podman, - ## set DOCKER_HOST or mount /var/run/docker.sock into the worker. - ## This worker is specialized for "native" jobs. Native jobs run in-process and thus are much more lightweight than other jobs windmill_worker_native: # Use ghcr.io/windmill-labs/windmill-ee:main for the ee diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index a5829bb2a2..2eb05a9bd5 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -268,6 +268,15 @@ export const settings: Record = { placeholder: '800', storage: 'setting' }, + { + label: 'Docker image storage size (MB)', + key: 'docker_image_storage_size_mb', + description: + 'Soft size cap for the per-job rootless-podman image store of # docker jobs (image layers + container writes). The worker polls the store and, past this size, logs an error and aborts the job's docker runtime — so a job cannot fill the worker disk with images, the way other languages are capped by the nsjail tmpfs. Soft: a job may overshoot by up to one poll interval (~2s) of writes. Route heavy docker jobs to workers with more disk via the docker tag. When left empty, defaults to 8192MB; set to 0 to disable the cap (bounded only by host disk).', + fieldType: 'number', + placeholder: '8192', + storage: 'setting' + }, { label: 'Default timeout', key: 'job_default_timeout',