diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index b4256f546d..6b5c113021 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -80c39bf7f3bfeda4cf0974ce32826f53affd9574 +d45b9a6cbe40f7fe5d322c850c50f64a6980e4f0 diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 8192f186d0..732b9cdc30 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -63,6 +63,10 @@ pub const SANDBOX_IMAGE_CACHE_MAX_MB_SETTING: &str = "sandbox_image_cache_max_mb pub const SANDBOX_IMAGE_PULL_POLICY_SETTING: &str = "sandbox_image_pull_policy"; pub const SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING: &str = "sandbox_image_default_registry"; pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth"; +// Enables the `#ssh ` directive that reroutes bash execution to a +// remote host over SSH (enterprise feature). Off by default. See +// windmill-worker/src/ssh_executor_ee.rs. +pub const SSH_EXECUTION_SETTING: &str = "ssh_execution_enabled"; pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config"; pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret"; diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index c7816d7a70..e3bd7ef7ef 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -888,6 +888,47 @@ impl BashAnnotations { } None } + + /// If the script declares `#ssh ` (a resource path after the + /// ssh annotation), returns that path. This reroutes execution to a remote + /// host over SSH (enterprise feature): the script runs on the host described + /// by the `ssh_target` resource at `` instead of on the worker. + /// The `#ssh $` form returns the `$`-prefixed token verbatim; the + /// executor resolves it from the job argument of that name at run time. + /// + /// Mirrors `sandbox_image`: only leading comment lines are scanned, stopping + /// at the first non-comment line. A bare `#ssh` with no path returns `None`. + /// Only an exact `#ssh ` line triggers the reroute — the target must + /// be a resource path (`u/...`/`f/...`) or a `$arg` reference (a valid + /// identifier), with nothing else on the line — so prose comments like + /// `# ssh into the box`, `# ssh tunnel/proxy setup is below` or + /// `# ssh $HOST manually first` never match. + pub fn ssh_target(code: &str) -> Option { + for line in code.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + if !line.starts_with('#') { + break; + } + let mut tokens = line[1..].split_whitespace(); + if tokens.next() == Some("ssh") { + if let Some(path) = tokens.next() { + let is_path = path.starts_with("u/") || path.starts_with("f/"); + let is_arg = path.strip_prefix('$').is_some_and(|a| { + !a.is_empty() + && !a.starts_with(|c: char| c.is_ascii_digit()) + && a.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') + }); + if (is_path || is_arg) && tokens.next().is_none() { + return Some(path.to_string()); + } + } + } + } + None + } } #[derive(Debug, Clone, Copy, PartialEq)] @@ -2283,6 +2324,60 @@ mod tests { ); } + #[test] + fn test_bash_ssh_target_annotation() { + // `#ssh ` returns the resource path (reroutes to remote execution). + assert_eq!( + BashAnnotations::ssh_target("#ssh f/infra/jump_node\nset -e\ndf -h"), + Some("f/infra/jump_node".to_string()) + ); + // `# ssh ` with a space after `#` also works. + assert_eq!( + BashAnnotations::ssh_target("# ssh u/me/box\n"), + Some("u/me/box".to_string()) + ); + // `#ssh $arg` (dynamic target from a job argument) is returned verbatim. + assert_eq!( + BashAnnotations::ssh_target("#ssh $jump_host\necho hi"), + Some("$jump_host".to_string()) + ); + // A bare `#ssh` with no path -> None. + assert_eq!(BashAnnotations::ssh_target("#ssh\necho hi"), None); + // `ssh` must be its own token, not a prefix. + assert_eq!(BashAnnotations::ssh_target("# sshd restart"), None); + // Prose comments mentioning ssh must not trigger the reroute: the line + // must be exactly `#ssh ` with a `u/`/`f/` path or `$identifier`. + assert_eq!( + BashAnnotations::ssh_target("# ssh into the box and restart nginx\necho hi"), + None + ); + assert_eq!( + BashAnnotations::ssh_target("# ssh tunnel/proxy setup is below\necho hi"), + None + ); + assert_eq!( + BashAnnotations::ssh_target("# ssh $HOST manually first\necho hi"), + None + ); + // Trailing tokens after a real-looking target also disqualify the line. + assert_eq!( + BashAnnotations::ssh_target("#ssh f/infra/box then reboot\necho hi"), + None + ); + // ...but a real directive on a later comment line is still found. + assert_eq!( + BashAnnotations::ssh_target("# ssh into the box\n#ssh f/infra/box\necho hi"), + Some("f/infra/box".to_string()) + ); + // Stops at the first non-comment line (declared too late is ignored). + assert_eq!( + BashAnnotations::ssh_target("echo hi\n#ssh f/infra/box"), + None + ); + // No annotation -> None (normal local bash). + assert_eq!(BashAnnotations::ssh_target("echo hello"), None); + } + #[test] fn test_mixed_tags() { let input = vec![ diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 4c470b8dfc..3101541d36 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -98,6 +98,26 @@ pub async fn handle_bash_job( .await; } + // `#ssh ` reroutes execution to a remote host over SSH + // (enterprise feature). The script runs on the host described by the + // `ssh_target` resource instead of on this worker. The OSS build returns a + // clear "enterprise feature" error from the stub. + if let Some(ssh_path) = windmill_common::worker::BashAnnotations::ssh_target(content) { + return crate::ssh_executor_oss::handle_ssh_bash_job( + &ssh_path, + mem_peak, + canceled_by, + job, + conn, + client, + content, + job_dir, + worker_name, + occupancy_metrics, + ) + .await; + } + // Check if sandbox annotation is used but nsjail is not available if annotation.sandbox && NSJAIL_AVAILABLE.is_none() { return Err(Error::ExecutionErr( diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 7727982cf8..f8b3edd069 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -75,6 +75,9 @@ mod sanitized_sql_params; mod schema; mod sql_s3_input; pub mod sql_utils; +#[cfg(feature = "private")] +mod ssh_executor_ee; +mod ssh_executor_oss; mod universal_pkg_installer; #[cfg(feature = "private")] mod volume_ee; diff --git a/backend/windmill-worker/src/ssh_executor_oss.rs b/backend/windmill-worker/src/ssh_executor_oss.rs new file mode 100644 index 0000000000..3cb1203c0e --- /dev/null +++ b/backend/windmill-worker/src/ssh_executor_oss.rs @@ -0,0 +1,27 @@ +#[cfg(feature = "private")] +#[allow(unused)] +pub(crate) use crate::ssh_executor_ee::*; + +// OSS stub: the `#ssh ` directive is an enterprise feature. The real +// implementation lives in ssh_executor_ee.rs (compiled under the `private` +// feature). See examples/usecase/ssh-execution-wrapper/ for the userland +// (no-license) alternative. +#[cfg(not(feature = "private"))] +pub(crate) async fn handle_ssh_bash_job( + _ssh_path: &str, + _mem_peak: &mut i32, + _canceled_by: &mut Option, + _job: &windmill_queue::MiniPulledJob, + _conn: &windmill_common::worker::Connection, + _client: &windmill_common::client::AuthedClient, + _content: &str, + _job_dir: &str, + _worker_name: &str, + _occupancy_metrics: &mut crate::common::OccupancyMetrics, +) -> Result, windmill_common::error::Error> { + Err(windmill_common::error::Error::ExecutionErr( + "SSH execution (#ssh) is an enterprise feature. Use the enterprise image, or the userland \ + SSH wrapper in examples/usecase/ssh-execution-wrapper/." + .to_string(), + )) +} diff --git a/examples/usecase/ssh-execution-wrapper/README.md b/examples/usecase/ssh-execution-wrapper/README.md new file mode 100644 index 0000000000..61304cc332 --- /dev/null +++ b/examples/usecase/ssh-execution-wrapper/README.md @@ -0,0 +1,254 @@ +SSH execution +============= + +Run a self-contained script on a remote host that Windmill cannot place a worker +on, but can reach over SSH (a jump box / utility node). + +> ⚠️ **Read this first: prefer agent workers.** For almost every "run code in an +> isolated/segmented environment" need, the recommended answer is an **agent +> worker**. See [When to use what](#when-to-use-what) below. + +There are two ways to do this, sharing the same `ssh_target` resource type: + +1. **The `#ssh` directive (recommended, enterprise).** Write a *normal* bash + script and add one line — `#ssh ` — at the top. The worker + reroutes execution to the remote host with **full parity**: typed positional + args in, structured result out, live streamed logs, cancellation, and the + remote exit code fails the job. This is a first-class backend feature; see + [The `#ssh` directive](#the-ssh-directive) below. +2. **The userland wrapper (no license required).** A reusable Windmill script + (`ssh_exec.sh` / `ssh_exec.py`) that you *call*, passing your remote code as a + string argument. No backend changes, no license — but you lose the editor + experience and structured results. Use it when you can't run the enterprise + image. Documented in [The userland wrapper](#the-userland-wrapper). + +What's here +----------- + +| File | Purpose | +| --- | --- | +| `ssh_target.resource-type.json` | Resource type: `host`, `port`, `user`, `private_key` (secret), `host_pubkey`, `accept_unknown_host`. Shared by both approaches. | +| `ssh_exec.sh` | The userland wrapper as a Windmill **bash** script. | +| `ssh_exec.py` | Userland wrapper as a Windmill **python** script (interpreter-dispatch table). | + +The `#ssh` directive +-------------------- + +**Enable it (once, as a superadmin):** the feature is enterprise-gated and off by +default. Turn on the `ssh_execution_enabled` instance setting (Superadmin +settings), which requires a valid enterprise license. + +**Use it:** create an `ssh_target` resource (see [Setup](#setup)), then write a +bash script with the directive on a leading comment line: + +```bash +#ssh f/infra/jump_node +# ^ reroutes this script to run on the host described by the +# ssh_target resource at f/infra/jump_node + +Service="$1" # typed positional args work as usual + +systemctl is-active "$Service" +echo "{\"service\": \"$Service\", \"checked\": true}" # last stdout line = result +``` + +The script runs on the remote host exactly as a local bash script would: the +arguments come from the run form, the result is collected the same way +(`result.json` > `result.out` > last stdout line), logs stream live, and a +non-zero remote exit fails the job. Only the *execution location* changes. + +**Dynamic target (`#ssh $`):** instead of hardcoding a path, the +directive can name a job argument that supplies the target at call time — for +picking the host from the run form, or fanning out over hosts in a flow forloop: + +```bash +#ssh $jump_host + +target="$1" # jump_host's position: always received as an empty string +df -h +``` + +The argument must be an `ssh_target` resource **path string** (with or without +the `$res:` prefix) — inline `ssh_target` objects are rejected, so the target is +always resolved through the runner's resource permissions and a caller can only +route execution to hosts whose resource they can read. Two things to note: the +target argument itself is forwarded to the remote script as an **empty string** +(its resolved value embeds the private key, which must never reach the remote +command line — its position is kept so the other `$1..$n` stay aligned), and +with a dynamic target the *runner* chooses where the code executes (bounded by +those resource permissions), whereas a hardcoded path lets the script author +pin it. + +**Host-key pinning** is enforced (`StrictHostKeyChecking=yes`) whenever the +resource's `host_pubkey` is set. An empty `host_pubkey` refuses to run unless the +resource explicitly sets `accept_unknown_host: true`, which falls back to weaker +TOFU (`accept-new`) and logs a warning — development only. + +**Parity boundary:** the remote receives the script body and its positional args +only. The Windmill runtime is *not* forwarded — `BASE_INTERNAL_URL`, the `wmill` +client, and reserved `WM_*` variables are unavailable remotely, so in-script +Windmill API callbacks won't work. The same trade-offs as the wrapper apply: +no remote dependency management, no nsjail sandbox, no S3 cache, per-job SSH +overhead. v1 is bash-only. + +The userland wrapper +-------------------- + +The wrapper takes an `ssh_target` resource, a `script_content` string, and a +`language`, then: + +1. writes the private key to a `0600` temp file (and a job-local `known_hosts`), +2. opens a single SSH connection (no TTY), +3. streams the script body to the remote host's stdin, where a small bootstrap + `mktemp`s a file, `trap`s its removal on `EXIT`, runs it with the right + interpreter, and exits with the script's exit code, +4. streams stdout/stderr back live and **propagates the remote exit code** so a + failed remote script fails the Windmill job. + +``` +Windmill worker Remote jump node +┌────────────────────┐ ┌─────────────────────────────┐ +│ ssh_exec.sh │ ssh (no -t) │ sh -c │ +│ key → 0600 tmp │ ───────────────▶ │ f=$(mktemp) │ +│ known_hosts pin │ body on stdin │ trap 'rm -f $f' EXIT │ +│ printf body | ssh │ ────────────────▶│ cat > $f │ +│ │ ◀─────────────── │ $f (live logs) │ +│ exit = ssh rc │ remote rc │ exit $? │ +└────────────────────┘ └─────────────────────────────┘ +``` + +Setup +----- + +1. **Create the resource type.** Push it with the CLI: + + ```bash + wmill resource-type push ssh_target.resource-type.json + ``` + + or recreate it in the UI (Resources → Resource Types) with the same schema. + `private_key` is marked secret (`"password": true`); `host_pubkey` is optional. + +2. **Create an `ssh_target` resource** for your jump node. Get `host_pubkey` from + the server (the `keytype key` portion, comment optional): + + ```bash + ssh-keyscan -t ed25519 your.jump.host # → ssh-ed25519 AAAAC3Nz... + ``` + +3. **Create a script** from `ssh_exec.sh` (bash) or `ssh_exec.py` (python). Mark + the first argument as a resource of type `ssh_target`. + +Usage +----- + +Call the wrapper with the target, the remote script body, and its language: + +```jsonc +{ + "ssh_target": "$res:u/me/my_jump_node", + "script_content": "set -euo pipefail\ndf -h\nsystemctl is-active nginx", + "language": "bash" +} +``` + +```jsonc +{ + "ssh_target": "$res:u/me/my_jump_node", + "script_content": "import platform\nprint(platform.platform())", + "language": "python" +} +``` + +Supported `language` keys: `bash`, `sh`, `python`/`python3`, `node`/`javascript`, +`ruby`, `php`, `perl`. Any other value is passed through as a raw remote +interpreter command. The remote host must already have that interpreter and any +dependencies installed (see tradeoffs). + +Design notes (the details that make or break it) +------------------------------------------------ + +These are deliberate and worth preserving if you adapt the wrapper: + +- **Exit-code propagation.** `ssh host cmd` returns the *remote* exit code. The + bash wrapper reads it via `${PIPESTATUS[1]}` and re-`exit`s it; the python + wrapper raises on non-zero. A failed remote script fails the Windmill job. +- **No TTY.** We never pass `-t`/`-tt`. A TTY merges stdout and stderr and + mangles log capture. Enable `-tt` **only** for interactive remote prompts + (e.g. `sudo` asking for a password). +- **Live, unbuffered logs.** `python -u` is used for python; for chatty bash that + buffers when piped, wrap the remote interpreter with `stdbuf -oL` (edit the + dispatch table, e.g. `interp="stdbuf -oL bash"`). +- **Remote cleanup survives failure.** The `trap 'rm -f "$f"' EXIT` is set on the + **remote** side, inside the streamed bootstrap, so the temp file is removed + even if the script errors out. +- **Host-key pinning.** With `host_pubkey` set, the wrapper pins it into a + job-local `known_hosts` and enforces `StrictHostKeyChecking=yes` (non-default + ports use the `[host]:port` form). With it empty, the wrapper refuses to run + unless `accept_unknown_host: true` is set on the resource, which falls back to + `accept-new` (TOFU) and warns — weaker; pin in production. +- **Quoted heredoc.** The remote bootstrap is built with `<<'REMOTE'` so `$f`, + `$?`, `$TMPDIR` are evaluated *remotely*, not expanded on the worker. +- **Body via stdin.** The script body is streamed over stdin, never written to a + local temp file or interpolated into the command line. +- **`--` before the destination.** OpenSSH parses a destination starting with + `-` as an option, so without the separator a crafted `user` like + `-oProxyCommand=...` in the resource would execute a local command on the + worker before host-key validation. Keep the `--` if you adapt the wrapper. +- **Multiple round-trips?** This wrapper makes a single SSH connection. If you + extend it to several `ssh` calls, add + `-o ControlMaster=auto -o ControlPersist=60 -o ControlPath=` to + reuse one connection instead of re-authenticating each time. + +When to use what +---------------- + +**Default: agent workers.** A Windmill *agent worker* is a lightweight worker +that runs *inside* the target environment and connects back to the Windmill +server over **outbound HTTP only** (using an `jwt_agent_*` token) — no inbound +ports, no DB access. It keeps everything Windmill workers normally give you: +automatic dependency management, nsjail sandboxing, the S3 binary cache, native +secrets, all languages, and no per-job connection overhead. If you can run a +process in the target environment, use an agent worker. + +**Worker-group tags** are the right tool when you *can* place a full worker in +the environment and want to route specific scripts to it. + +**This SSH wrapper** is for the narrow case where **both** are true: + +- you can only reach a **jump/utility node** over SSH (you cannot place any + worker or agent process there), and +- the scripts are **simple and self-contained** (no Windmill-managed deps). + +What you lose with the SSH path +------------------------------- + +- **No dependency management.** The remote host must already have the interpreter + *and* every library/tool the script uses. Nothing is installed or locked. +- **No nsjail sandboxing.** The script runs as the SSH user with that user's full + privileges. The jump node becomes a high-value target — scope the key and user + tightly. +- **No S3 / binary cache.** No shared cache of dependencies or artifacts. +- **Per-job SSH overhead.** Each run pays connection + auth latency (mitigable + with ControlMaster only if you make multiple round-trips). +- **No native Windmill integrations on the remote side** — no resource/variable + injection, no `wmill` client, no flow step context beyond what you pass in. + +Limitations of this prototype +----------------------------- + +- Requires an `ssh` client (and `jq` for the bash variant) on the worker. +- Assumes a self-contained, non-interactive script. No stdin is forwarded to the + remote script (stdin carries the script body). +- Unknown `language` values are passed through verbatim as the remote + interpreter — keep `language` author-controlled, not end-user input. + +Tested +------ + +Both wrappers were exercised against a local `sshd`: success path, remote +exit-code propagation (bash `${PIPESTATUS[1]}`, python raises), clean +stdout/stderr separation, `python -u` interpreter dispatch, host-key pinning +rejecting a wrong key (script never runs), the TOFU opt-in +(`accept_unknown_host: true`) and the refusal without it, and confirmed remote +*and* local temp-file cleanup. diff --git a/examples/usecase/ssh-execution-wrapper/ssh_exec.py b/examples/usecase/ssh-execution-wrapper/ssh_exec.py new file mode 100644 index 0000000000..6bd99be2a4 --- /dev/null +++ b/examples/usecase/ssh-execution-wrapper/ssh_exec.py @@ -0,0 +1,116 @@ +# Windmill SSH execution wrapper (python) +# -------------------------------------------------------------------------- +# Python variant of ssh_exec.sh. Same contract: run a self-contained script on +# a remote host over SSH, stream stdout/stderr live, and fail the Windmill job +# when the remote script fails. Userland prototype, no backend changes. +# +# Arguments: +# ssh_target resource of type `ssh_target` (received as a dict) +# script_content the body of the script to run on the remote host +# language interpreter key: bash|sh|python|node|ruby|php|perl +# (default: bash; anything else is treated as a raw remote +# interpreter command) +# +# Worker requirements: an `ssh` client installed on the worker. + +import os +import subprocess +import sys +import tempfile + +# `python3 -u` forces unbuffered output so logs stream live. +INTERPRETERS = { + "bash": "bash", + "sh": "sh", + "python": "python3 -u", + "python3": "python3 -u", + "node": "node", + "javascript": "node", + "ruby": "ruby", + "php": "php", + "perl": "perl", +} + +# Single-quoted (raw) so $f / $? / $TMPDIR are evaluated remotely, not here. +# @@INTERP@@ is replaced with the chosen interpreter before sending. +REMOTE_BOOTSTRAP = ( + "set -u\n" + 'f=$(mktemp "${TMPDIR:-/tmp}/wmssh_job.XXXXXX") || exit 1\n' + "trap 'rm -f \"$f\"' EXIT\n" # remote-side cleanup, survives script failure + 'cat >"$f"\n' # read the streamed script body from stdin + '@@INTERP@@ "$f"\n' # execute with the chosen interpreter + "exit $?\n" # propagate the remote exit code +) + + +def main(ssh_target: dict, script_content: str, language: str = "bash"): + host = ssh_target["host"] + user = ssh_target["user"] + private_key = ssh_target["private_key"] + port = str(ssh_target.get("port") or 22) + host_pubkey = (ssh_target.get("host_pubkey") or "").strip() + accept_unknown_host = bool(ssh_target.get("accept_unknown_host")) + + interp = INTERPRETERS.get(language, language) # passthrough for unknown keys + + # 0600 temp files for the key and a job-local known_hosts. + keyfile = tempfile.NamedTemporaryFile("w", delete=False) + known_hosts = tempfile.NamedTemporaryFile("w", delete=False) + try: + keyfile.write(private_key.rstrip("\n") + "\n") # trailing newline required by some keys + keyfile.close() + os.chmod(keyfile.name, 0o600) + + ssh_opts = [ + "-o", "BatchMode=yes", + "-o", "ConnectTimeout=15", + "-o", f"UserKnownHostsFile={known_hosts.name}", + "-p", port, + "-i", keyfile.name, + ] + + if host_pubkey: + # Pin the server key; non-default ports use the [host]:port form. + entry = f"{host} {host_pubkey}" if port == "22" else f"[{host}]:{port} {host_pubkey}" + known_hosts.write(entry + "\n") + known_hosts.close() + ssh_opts += ["-o", "StrictHostKeyChecking=yes"] + elif accept_unknown_host: + known_hosts.close() + print( + "WARN: ssh_target.host_pubkey is empty; using TOFU (accept-new) " + "because accept_unknown_host=true. Pin host_pubkey for production.", + file=sys.stderr, + flush=True, + ) + ssh_opts += ["-o", "StrictHostKeyChecking=accept-new"] + else: + known_hosts.close() + raise ValueError( + "ssh_target.host_pubkey is empty. Pin the host key " + f"(ssh-keyscan -t ed25519 {host}) or set accept_unknown_host=true " + "to allow TOFU (insecure against MITM)." + ) + + remote = REMOTE_BOOTSTRAP.replace("@@INTERP@@", interp) + # no -t/-tt: stdout and stderr stay separate for clean log capture. + # `--` so a crafted user/host (e.g. "-oProxyCommand=...") can never be + # parsed as an ssh option + cmd = ["ssh", *ssh_opts, "--", f"{user}@{host}", remote] + + # stdout/stderr are inherited from this process so they stream live to + # the Windmill job log; only stdin is a pipe for the script body. + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE) + proc.communicate(input=(script_content + "\n").encode()) + rc = proc.returncode + finally: + for path in (keyfile.name, known_hosts.name): + try: + os.unlink(path) + except OSError: + pass + + # Raise on non-zero so the Windmill job fails with the remote exit code. + if rc != 0: + raise RuntimeError(f"Remote script exited with code {rc}") + return {"ok": True, "exit_code": 0, "host": host} diff --git a/examples/usecase/ssh-execution-wrapper/ssh_exec.sh b/examples/usecase/ssh-execution-wrapper/ssh_exec.sh new file mode 100755 index 0000000000..5c23402d1d --- /dev/null +++ b/examples/usecase/ssh-execution-wrapper/ssh_exec.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Windmill SSH execution wrapper (bash) +# -------------------------------------------------------------------------- +# Runs a self-contained script on a remote host reachable over SSH from the +# Windmill worker, streaming stdout/stderr back live and propagating the +# REMOTE exit code so a failed remote script fails the Windmill job. +# +# This is a userland prototype: it makes NO backend changes. See README.md for +# when to use it (only-a-jump-node reachable, self-contained scripts) and why +# agent workers are the recommended default. +# +# Windmill positional bash arguments: +# 1. ssh_target resource of type `ssh_target` (Windmill passes it as JSON) +# 2. script_content the body of the script to run on the remote host +# 3. language interpreter key: bash|sh|python|node|ruby|php|perl +# (default: bash; anything else is treated as a raw remote +# interpreter command) +# +# Worker requirements: `ssh` client and `jq` must be installed on the worker. + +ssh_target="$1" +script_content="$2" +language="${3:-bash}" + +set -euo pipefail +umask 077 + +command -v jq >/dev/null || { echo "FATAL: jq is required on the worker" >&2; exit 127; } +command -v ssh >/dev/null || { echo "FATAL: ssh client is required on the worker" >&2; exit 127; } + +# --- parse the ssh_target resource ----------------------------------------- +# `jq -e` exits non-zero when the field is null/absent, so a missing required +# field fails fast with a clear message. +host=$(jq -er '.host' <<<"$ssh_target") || { echo "FATAL: ssh_target.host missing" >&2; exit 2; } +user=$(jq -er '.user' <<<"$ssh_target") || { echo "FATAL: ssh_target.user missing" >&2; exit 2; } +private_key=$(jq -er '.private_key' <<<"$ssh_target") || { echo "FATAL: ssh_target.private_key missing" >&2; exit 2; } +port=$(jq -r '.port // 22' <<<"$ssh_target") +host_pubkey=$(jq -r '.host_pubkey // empty' <<<"$ssh_target") +accept_unknown_host=$(jq -r '.accept_unknown_host // false' <<<"$ssh_target") + +# --- interpreter dispatch table -------------------------------------------- +# `python3 -u` forces unbuffered output so logs stream live. For chatty bash +# scripts that buffer, wrap the remote interpreter with `stdbuf -oL` (see README). +case "$language" in + bash) interp="bash" ;; + sh) interp="sh" ;; + python|python3) interp="python3 -u" ;; + node|javascript) interp="node" ;; + ruby) interp="ruby" ;; + php) interp="php" ;; + perl) interp="perl" ;; + *) interp="$language" ;; # passthrough: raw remote interpreter command +esac + +# --- private key -> 0600 temp file + job-local known_hosts ------------------ +keyfile=$(mktemp "${TMPDIR:-/tmp}/wmssh_key.XXXXXX") +known_hosts=$(mktemp "${TMPDIR:-/tmp}/wmssh_kh.XXXXXX") +cleanup() { rm -f "$keyfile" "$known_hosts"; } +trap cleanup EXIT + +printf '%s\n' "$private_key" >"$keyfile" # trailing newline: some keys require it +chmod 600 "$keyfile" + +# --- host-key handling ------------------------------------------------------ +ssh_opts=( + -o BatchMode=yes + -o ConnectTimeout=15 + -o "UserKnownHostsFile=$known_hosts" + -p "$port" + -i "$keyfile" +) +if [ -n "$host_pubkey" ]; then + # Pin the server key: non-default ports use the `[host]:port` known_hosts form. + if [ "$port" = "22" ]; then + printf '%s %s\n' "$host" "$host_pubkey" >"$known_hosts" + else + printf '[%s]:%s %s\n' "$host" "$port" "$host_pubkey" >"$known_hosts" + fi + ssh_opts+=( -o StrictHostKeyChecking=yes ) +elif [ "$accept_unknown_host" = "true" ]; then + echo "WARN: ssh_target.host_pubkey is empty; using TOFU (accept-new) because accept_unknown_host=true. Pin host_pubkey for production." >&2 + ssh_opts+=( -o StrictHostKeyChecking=accept-new ) +else + echo "FATAL: ssh_target.host_pubkey is empty. Pin the host key (ssh-keyscan -t ed25519 $host) or set accept_unknown_host=true to allow TOFU (insecure against MITM)." >&2 + exit 2 +fi + +# --- remote bootstrap (runs ON the remote host) ---------------------------- +# Single-quoted heredoc: NOTHING is expanded locally. $f / $? / $TMPDIR are +# all evaluated remotely. The script body arrives on the remote's stdin. +remote_bootstrap=$(cat <<'REMOTE' +set -u +f=$(mktemp "${TMPDIR:-/tmp}/wmssh_job.XXXXXX") || exit 1 +trap 'rm -f "$f"' EXIT # remote-side cleanup, survives script failure +cat >"$f" # read the streamed script body from stdin +@@INTERP@@ "$f" # execute with the chosen interpreter +exit $? # propagate the remote exit code +REMOTE +) +# NOTE: `interp` is only controlled for the known dispatch keys above — the +# `*)` case passes `language` through verbatim into the remote bootstrap, so +# keep `language` author-controlled, never end-user input (see README). +remote_bootstrap=${remote_bootstrap//@@INTERP@@/$interp} + +# --- execute --------------------------------------------------------------- +# - no -t/-tt: keep stdout and stderr separate for clean log capture +# (add `-tt` ONLY when you need an interactive TTY, e.g. sudo password prompts) +# - body is streamed via stdin so it never touches a local temp file +# - PIPESTATUS[1] is ssh's exit code == the remote script's exit code +set +e +printf '%s\n' "$script_content" | ssh "${ssh_opts[@]}" -- "$user@$host" "$remote_bootstrap" +rc=${PIPESTATUS[1]} +exit "$rc" diff --git a/examples/usecase/ssh-execution-wrapper/ssh_target.resource-type.json b/examples/usecase/ssh-execution-wrapper/ssh_target.resource-type.json new file mode 100644 index 0000000000..163a554623 --- /dev/null +++ b/examples/usecase/ssh-execution-wrapper/ssh_target.resource-type.json @@ -0,0 +1,38 @@ +{ + "schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "host": { + "type": "string", + "description": "Hostname or IP of the SSH jump/utility node to execute on." + }, + "port": { + "type": "integer", + "description": "SSH port.", + "default": 22 + }, + "user": { + "type": "string", + "description": "SSH login user." + }, + "private_key": { + "type": "string", + "description": "PEM-encoded private key used for authentication. Stored as a secret.", + "password": true + }, + "host_pubkey": { + "type": "string", + "description": "Server host public key line for known_hosts pinning, e.g. 'ssh-ed25519 AAAAC3Nz...' (get it with `ssh-keyscan -t ed25519 `). When set, the wrapper enforces StrictHostKeyChecking=yes. When empty, execution is refused unless accept_unknown_host is true.", + "default": "" + }, + "accept_unknown_host": { + "type": "boolean", + "description": "Allow connecting without a pinned host_pubkey by trusting the host key on first use (StrictHostKeyChecking=accept-new). Insecure against MITM — development only.", + "default": false + } + }, + "required": ["host", "user", "private_key"] + }, + "description": "Target host reachable over SSH for the lightweight SSH execution wrapper. See examples/usecase/ssh-execution-wrapper/README.md." +} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 10a0519d7e..6af6926c4f 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -322,6 +322,16 @@ export const settings: Record = { '{\n "auths": {\n "myregistry.example.com": {\n "auth": "BASE64(username:password)"\n }\n }\n}', storage: 'setting' }, + { + label: 'SSH execution (#ssh)', + key: 'ssh_execution_enabled', + fieldType: 'boolean', + description: + 'Allow bash scripts starting with a #ssh <resource_path> directive to run on the remote host described by the referenced ssh_target resource instead of the worker. Off by default.', + storage: 'setting', + ee_only: '', + hideInQuickSetup: true + }, { label: 'Default timeout', key: 'job_default_timeout',