mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-08-18 08:01:26 +00:00
feat(worker): #ssh directive to run a bash script on a remote SSH host (#9479)
* feat(worker): #ssh directive to run a bash script on a remote SSH host Add a first-class `#ssh <resource_path>` bash directive that reroutes a normal bash script to run on a remote host reached over SSH (a jump/utility node) instead of on the worker, with full parity: typed positional args in, structured result out, live streamed logs, cancellation, and remote exit-code propagation. It mirrors the existing `# sandbox <image>` precedent: the directive is parsed in handle_bash_job and reroutes to a specialized handler that reuses handle_child for all execution plumbing. - windmill-common: BashAnnotations::ssh_target() parser (+ unit test) and the ssh_execution_enabled instance setting (off by default) - windmill-worker: reroute hook in bash_executor + ssh_executor_oss shim. OSS returns a clear "enterprise feature" error; the real handler lives in ssh_executor_ee.rs (private feature) and is gated by a valid enterprise license + the instance setting. - examples/usecase/ssh-execution-wrapper: the ssh_target resource type, a userland wrapper (no-license fallback), and a README documenting both paths and the trade-offs vs agent workers. EE companion: windmill-labs/windmill-ee-private (ee-repo-ref.txt bumped). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> * feat(worker): ssh host-key opt-in, 0600 key write, instance setting UI * chore: update ee-repo-ref * feat(worker): #ssh $arg form to take the ssh target from a job argument * fix(worker): #ssh token must look like a target; $arg restricted to path strings * fix(worker): tighten #ssh parser to exact directive; add -- ssh destination guard * chore: update ee-repo-ref to d45b9a6cbe40f7fe5d322c850c50f64a6980e4f0 This commit updates the EE repository reference after PR #609 was merged in windmill-ee-private. Previous ee-repo-ref: 2804f1aa8e74b3a7733aeb6f5044d5085193872a New ee-repo-ref: d45b9a6cbe40f7fe5d322c850c50f64a6980e4f0 Automated by sync-ee-ref workflow. --------- Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
This commit is contained in:
Ruben Fiszel
GitHub
@@ -1 +1 @@
|
||||
80c39bf7f3bfeda4cf0974ce32826f53affd9574
|
||||
d45b9a6cbe40f7fe5d322c850c50f64a6980e4f0
|
||||
|
||||
@@ -63,6 +63,10 @@ pub const SANDBOX_IMAGE_CACHE_MAX_MB_SETTING: &str = "sandbox_image_cache_max_mb
|
||||
pub const SANDBOX_IMAGE_PULL_POLICY_SETTING: &str = "sandbox_image_pull_policy";
|
||||
pub const SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING: &str = "sandbox_image_default_registry";
|
||||
pub const SANDBOX_REGISTRY_AUTH_SETTING: &str = "sandbox_registry_auth";
|
||||
// Enables the `#ssh <resource>` directive that reroutes bash execution to a
|
||||
// remote host over SSH (enterprise feature). Off by default. See
|
||||
// windmill-worker/src/ssh_executor_ee.rs.
|
||||
pub const SSH_EXECUTION_SETTING: &str = "ssh_execution_enabled";
|
||||
pub const OBJECT_STORE_CONFIG_SETTING: &str = "object_store_cache_config";
|
||||
pub const HUB_API_SECRET_SETTING: &str = "hub_api_secret";
|
||||
|
||||
|
||||
@@ -888,6 +888,47 @@ impl BashAnnotations {
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// If the script declares `#ssh <resource_path>` (a resource path after the
|
||||
/// ssh annotation), returns that path. This reroutes execution to a remote
|
||||
/// host over SSH (enterprise feature): the script runs on the host described
|
||||
/// by the `ssh_target` resource at `<resource_path>` instead of on the worker.
|
||||
/// The `#ssh $<arg_name>` form returns the `$`-prefixed token verbatim; the
|
||||
/// executor resolves it from the job argument of that name at run time.
|
||||
///
|
||||
/// Mirrors `sandbox_image`: only leading comment lines are scanned, stopping
|
||||
/// at the first non-comment line. A bare `#ssh` with no path returns `None`.
|
||||
/// Only an exact `#ssh <target>` line triggers the reroute — the target must
|
||||
/// be a resource path (`u/...`/`f/...`) or a `$arg` reference (a valid
|
||||
/// identifier), with nothing else on the line — so prose comments like
|
||||
/// `# ssh into the box`, `# ssh tunnel/proxy setup is below` or
|
||||
/// `# ssh $HOST manually first` never match.
|
||||
pub fn ssh_target(code: &str) -> Option<String> {
|
||||
for line in code.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !line.starts_with('#') {
|
||||
break;
|
||||
}
|
||||
let mut tokens = line[1..].split_whitespace();
|
||||
if tokens.next() == Some("ssh") {
|
||||
if let Some(path) = tokens.next() {
|
||||
let is_path = path.starts_with("u/") || path.starts_with("f/");
|
||||
let is_arg = path.strip_prefix('$').is_some_and(|a| {
|
||||
!a.is_empty()
|
||||
&& !a.starts_with(|c: char| c.is_ascii_digit())
|
||||
&& a.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||
});
|
||||
if (is_path || is_arg) && tokens.next().is_none() {
|
||||
return Some(path.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
@@ -2283,6 +2324,60 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bash_ssh_target_annotation() {
|
||||
// `#ssh <path>` returns the resource path (reroutes to remote execution).
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("#ssh f/infra/jump_node\nset -e\ndf -h"),
|
||||
Some("f/infra/jump_node".to_string())
|
||||
);
|
||||
// `# ssh <path>` with a space after `#` also works.
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("# ssh u/me/box\n"),
|
||||
Some("u/me/box".to_string())
|
||||
);
|
||||
// `#ssh $arg` (dynamic target from a job argument) is returned verbatim.
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("#ssh $jump_host\necho hi"),
|
||||
Some("$jump_host".to_string())
|
||||
);
|
||||
// A bare `#ssh` with no path -> None.
|
||||
assert_eq!(BashAnnotations::ssh_target("#ssh\necho hi"), None);
|
||||
// `ssh` must be its own token, not a prefix.
|
||||
assert_eq!(BashAnnotations::ssh_target("# sshd restart"), None);
|
||||
// Prose comments mentioning ssh must not trigger the reroute: the line
|
||||
// must be exactly `#ssh <target>` with a `u/`/`f/` path or `$identifier`.
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("# ssh into the box and restart nginx\necho hi"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("# ssh tunnel/proxy setup is below\necho hi"),
|
||||
None
|
||||
);
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("# ssh $HOST manually first\necho hi"),
|
||||
None
|
||||
);
|
||||
// Trailing tokens after a real-looking target also disqualify the line.
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("#ssh f/infra/box then reboot\necho hi"),
|
||||
None
|
||||
);
|
||||
// ...but a real directive on a later comment line is still found.
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("# ssh into the box\n#ssh f/infra/box\necho hi"),
|
||||
Some("f/infra/box".to_string())
|
||||
);
|
||||
// Stops at the first non-comment line (declared too late is ignored).
|
||||
assert_eq!(
|
||||
BashAnnotations::ssh_target("echo hi\n#ssh f/infra/box"),
|
||||
None
|
||||
);
|
||||
// No annotation -> None (normal local bash).
|
||||
assert_eq!(BashAnnotations::ssh_target("echo hello"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_mixed_tags() {
|
||||
let input = vec![
|
||||
|
||||
@@ -98,6 +98,26 @@ pub async fn handle_bash_job(
|
||||
.await;
|
||||
}
|
||||
|
||||
// `#ssh <resource_path>` reroutes execution to a remote host over SSH
|
||||
// (enterprise feature). The script runs on the host described by the
|
||||
// `ssh_target` resource instead of on this worker. The OSS build returns a
|
||||
// clear "enterprise feature" error from the stub.
|
||||
if let Some(ssh_path) = windmill_common::worker::BashAnnotations::ssh_target(content) {
|
||||
return crate::ssh_executor_oss::handle_ssh_bash_job(
|
||||
&ssh_path,
|
||||
mem_peak,
|
||||
canceled_by,
|
||||
job,
|
||||
conn,
|
||||
client,
|
||||
content,
|
||||
job_dir,
|
||||
worker_name,
|
||||
occupancy_metrics,
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
// Check if sandbox annotation is used but nsjail is not available
|
||||
if annotation.sandbox && NSJAIL_AVAILABLE.is_none() {
|
||||
return Err(Error::ExecutionErr(
|
||||
|
||||
@@ -75,6 +75,9 @@ mod sanitized_sql_params;
|
||||
mod schema;
|
||||
mod sql_s3_input;
|
||||
pub mod sql_utils;
|
||||
#[cfg(feature = "private")]
|
||||
mod ssh_executor_ee;
|
||||
mod ssh_executor_oss;
|
||||
mod universal_pkg_installer;
|
||||
#[cfg(feature = "private")]
|
||||
mod volume_ee;
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
#[cfg(feature = "private")]
|
||||
#[allow(unused)]
|
||||
pub(crate) use crate::ssh_executor_ee::*;
|
||||
|
||||
// OSS stub: the `#ssh <resource>` directive is an enterprise feature. The real
|
||||
// implementation lives in ssh_executor_ee.rs (compiled under the `private`
|
||||
// feature). See examples/usecase/ssh-execution-wrapper/ for the userland
|
||||
// (no-license) alternative.
|
||||
#[cfg(not(feature = "private"))]
|
||||
pub(crate) async fn handle_ssh_bash_job(
|
||||
_ssh_path: &str,
|
||||
_mem_peak: &mut i32,
|
||||
_canceled_by: &mut Option<windmill_queue::CanceledBy>,
|
||||
_job: &windmill_queue::MiniPulledJob,
|
||||
_conn: &windmill_common::worker::Connection,
|
||||
_client: &windmill_common::client::AuthedClient,
|
||||
_content: &str,
|
||||
_job_dir: &str,
|
||||
_worker_name: &str,
|
||||
_occupancy_metrics: &mut crate::common::OccupancyMetrics,
|
||||
) -> Result<Box<serde_json::value::RawValue>, windmill_common::error::Error> {
|
||||
Err(windmill_common::error::Error::ExecutionErr(
|
||||
"SSH execution (#ssh) is an enterprise feature. Use the enterprise image, or the userland \
|
||||
SSH wrapper in examples/usecase/ssh-execution-wrapper/."
|
||||
.to_string(),
|
||||
))
|
||||
}
|
||||
@@ -0,0 +1,254 @@
|
||||
SSH execution
|
||||
=============
|
||||
|
||||
Run a self-contained script on a remote host that Windmill cannot place a worker
|
||||
on, but can reach over SSH (a jump box / utility node).
|
||||
|
||||
> ⚠️ **Read this first: prefer agent workers.** For almost every "run code in an
|
||||
> isolated/segmented environment" need, the recommended answer is an **agent
|
||||
> worker**. See [When to use what](#when-to-use-what) below.
|
||||
|
||||
There are two ways to do this, sharing the same `ssh_target` resource type:
|
||||
|
||||
1. **The `#ssh` directive (recommended, enterprise).** Write a *normal* bash
|
||||
script and add one line — `#ssh <resource_path>` — at the top. The worker
|
||||
reroutes execution to the remote host with **full parity**: typed positional
|
||||
args in, structured result out, live streamed logs, cancellation, and the
|
||||
remote exit code fails the job. This is a first-class backend feature; see
|
||||
[The `#ssh` directive](#the-ssh-directive) below.
|
||||
2. **The userland wrapper (no license required).** A reusable Windmill script
|
||||
(`ssh_exec.sh` / `ssh_exec.py`) that you *call*, passing your remote code as a
|
||||
string argument. No backend changes, no license — but you lose the editor
|
||||
experience and structured results. Use it when you can't run the enterprise
|
||||
image. Documented in [The userland wrapper](#the-userland-wrapper).
|
||||
|
||||
What's here
|
||||
-----------
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `ssh_target.resource-type.json` | Resource type: `host`, `port`, `user`, `private_key` (secret), `host_pubkey`, `accept_unknown_host`. Shared by both approaches. |
|
||||
| `ssh_exec.sh` | The userland wrapper as a Windmill **bash** script. |
|
||||
| `ssh_exec.py` | Userland wrapper as a Windmill **python** script (interpreter-dispatch table). |
|
||||
|
||||
The `#ssh` directive
|
||||
--------------------
|
||||
|
||||
**Enable it (once, as a superadmin):** the feature is enterprise-gated and off by
|
||||
default. Turn on the `ssh_execution_enabled` instance setting (Superadmin
|
||||
settings), which requires a valid enterprise license.
|
||||
|
||||
**Use it:** create an `ssh_target` resource (see [Setup](#setup)), then write a
|
||||
bash script with the directive on a leading comment line:
|
||||
|
||||
```bash
|
||||
#ssh f/infra/jump_node
|
||||
# ^ reroutes this script to run on the host described by the
|
||||
# ssh_target resource at f/infra/jump_node
|
||||
|
||||
Service="$1" # typed positional args work as usual
|
||||
|
||||
systemctl is-active "$Service"
|
||||
echo "{\"service\": \"$Service\", \"checked\": true}" # last stdout line = result
|
||||
```
|
||||
|
||||
The script runs on the remote host exactly as a local bash script would: the
|
||||
arguments come from the run form, the result is collected the same way
|
||||
(`result.json` > `result.out` > last stdout line), logs stream live, and a
|
||||
non-zero remote exit fails the job. Only the *execution location* changes.
|
||||
|
||||
**Dynamic target (`#ssh $<arg_name>`):** instead of hardcoding a path, the
|
||||
directive can name a job argument that supplies the target at call time — for
|
||||
picking the host from the run form, or fanning out over hosts in a flow forloop:
|
||||
|
||||
```bash
|
||||
#ssh $jump_host
|
||||
|
||||
target="$1" # jump_host's position: always received as an empty string
|
||||
df -h
|
||||
```
|
||||
|
||||
The argument must be an `ssh_target` resource **path string** (with or without
|
||||
the `$res:` prefix) — inline `ssh_target` objects are rejected, so the target is
|
||||
always resolved through the runner's resource permissions and a caller can only
|
||||
route execution to hosts whose resource they can read. Two things to note: the
|
||||
target argument itself is forwarded to the remote script as an **empty string**
|
||||
(its resolved value embeds the private key, which must never reach the remote
|
||||
command line — its position is kept so the other `$1..$n` stay aligned), and
|
||||
with a dynamic target the *runner* chooses where the code executes (bounded by
|
||||
those resource permissions), whereas a hardcoded path lets the script author
|
||||
pin it.
|
||||
|
||||
**Host-key pinning** is enforced (`StrictHostKeyChecking=yes`) whenever the
|
||||
resource's `host_pubkey` is set. An empty `host_pubkey` refuses to run unless the
|
||||
resource explicitly sets `accept_unknown_host: true`, which falls back to weaker
|
||||
TOFU (`accept-new`) and logs a warning — development only.
|
||||
|
||||
**Parity boundary:** the remote receives the script body and its positional args
|
||||
only. The Windmill runtime is *not* forwarded — `BASE_INTERNAL_URL`, the `wmill`
|
||||
client, and reserved `WM_*` variables are unavailable remotely, so in-script
|
||||
Windmill API callbacks won't work. The same trade-offs as the wrapper apply:
|
||||
no remote dependency management, no nsjail sandbox, no S3 cache, per-job SSH
|
||||
overhead. v1 is bash-only.
|
||||
|
||||
The userland wrapper
|
||||
--------------------
|
||||
|
||||
The wrapper takes an `ssh_target` resource, a `script_content` string, and a
|
||||
`language`, then:
|
||||
|
||||
1. writes the private key to a `0600` temp file (and a job-local `known_hosts`),
|
||||
2. opens a single SSH connection (no TTY),
|
||||
3. streams the script body to the remote host's stdin, where a small bootstrap
|
||||
`mktemp`s a file, `trap`s its removal on `EXIT`, runs it with the right
|
||||
interpreter, and exits with the script's exit code,
|
||||
4. streams stdout/stderr back live and **propagates the remote exit code** so a
|
||||
failed remote script fails the Windmill job.
|
||||
|
||||
```
|
||||
Windmill worker Remote jump node
|
||||
┌────────────────────┐ ┌─────────────────────────────┐
|
||||
│ ssh_exec.sh │ ssh (no -t) │ sh -c <bootstrap> │
|
||||
│ key → 0600 tmp │ ───────────────▶ │ f=$(mktemp) │
|
||||
│ known_hosts pin │ body on stdin │ trap 'rm -f $f' EXIT │
|
||||
│ printf body | ssh │ ────────────────▶│ cat > $f │
|
||||
│ │ ◀─────────────── │ <interp> $f (live logs) │
|
||||
│ exit = ssh rc │ remote rc │ exit $? │
|
||||
└────────────────────┘ └─────────────────────────────┘
|
||||
```
|
||||
|
||||
Setup
|
||||
-----
|
||||
|
||||
1. **Create the resource type.** Push it with the CLI:
|
||||
|
||||
```bash
|
||||
wmill resource-type push ssh_target.resource-type.json
|
||||
```
|
||||
|
||||
or recreate it in the UI (Resources → Resource Types) with the same schema.
|
||||
`private_key` is marked secret (`"password": true`); `host_pubkey` is optional.
|
||||
|
||||
2. **Create an `ssh_target` resource** for your jump node. Get `host_pubkey` from
|
||||
the server (the `keytype key` portion, comment optional):
|
||||
|
||||
```bash
|
||||
ssh-keyscan -t ed25519 your.jump.host # → ssh-ed25519 AAAAC3Nz...
|
||||
```
|
||||
|
||||
3. **Create a script** from `ssh_exec.sh` (bash) or `ssh_exec.py` (python). Mark
|
||||
the first argument as a resource of type `ssh_target`.
|
||||
|
||||
Usage
|
||||
-----
|
||||
|
||||
Call the wrapper with the target, the remote script body, and its language:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"ssh_target": "$res:u/me/my_jump_node",
|
||||
"script_content": "set -euo pipefail\ndf -h\nsystemctl is-active nginx",
|
||||
"language": "bash"
|
||||
}
|
||||
```
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"ssh_target": "$res:u/me/my_jump_node",
|
||||
"script_content": "import platform\nprint(platform.platform())",
|
||||
"language": "python"
|
||||
}
|
||||
```
|
||||
|
||||
Supported `language` keys: `bash`, `sh`, `python`/`python3`, `node`/`javascript`,
|
||||
`ruby`, `php`, `perl`. Any other value is passed through as a raw remote
|
||||
interpreter command. The remote host must already have that interpreter and any
|
||||
dependencies installed (see tradeoffs).
|
||||
|
||||
Design notes (the details that make or break it)
|
||||
------------------------------------------------
|
||||
|
||||
These are deliberate and worth preserving if you adapt the wrapper:
|
||||
|
||||
- **Exit-code propagation.** `ssh host cmd` returns the *remote* exit code. The
|
||||
bash wrapper reads it via `${PIPESTATUS[1]}` and re-`exit`s it; the python
|
||||
wrapper raises on non-zero. A failed remote script fails the Windmill job.
|
||||
- **No TTY.** We never pass `-t`/`-tt`. A TTY merges stdout and stderr and
|
||||
mangles log capture. Enable `-tt` **only** for interactive remote prompts
|
||||
(e.g. `sudo` asking for a password).
|
||||
- **Live, unbuffered logs.** `python -u` is used for python; for chatty bash that
|
||||
buffers when piped, wrap the remote interpreter with `stdbuf -oL` (edit the
|
||||
dispatch table, e.g. `interp="stdbuf -oL bash"`).
|
||||
- **Remote cleanup survives failure.** The `trap 'rm -f "$f"' EXIT` is set on the
|
||||
**remote** side, inside the streamed bootstrap, so the temp file is removed
|
||||
even if the script errors out.
|
||||
- **Host-key pinning.** With `host_pubkey` set, the wrapper pins it into a
|
||||
job-local `known_hosts` and enforces `StrictHostKeyChecking=yes` (non-default
|
||||
ports use the `[host]:port` form). With it empty, the wrapper refuses to run
|
||||
unless `accept_unknown_host: true` is set on the resource, which falls back to
|
||||
`accept-new` (TOFU) and warns — weaker; pin in production.
|
||||
- **Quoted heredoc.** The remote bootstrap is built with `<<'REMOTE'` so `$f`,
|
||||
`$?`, `$TMPDIR` are evaluated *remotely*, not expanded on the worker.
|
||||
- **Body via stdin.** The script body is streamed over stdin, never written to a
|
||||
local temp file or interpolated into the command line.
|
||||
- **`--` before the destination.** OpenSSH parses a destination starting with
|
||||
`-` as an option, so without the separator a crafted `user` like
|
||||
`-oProxyCommand=...` in the resource would execute a local command on the
|
||||
worker before host-key validation. Keep the `--` if you adapt the wrapper.
|
||||
- **Multiple round-trips?** This wrapper makes a single SSH connection. If you
|
||||
extend it to several `ssh` calls, add
|
||||
`-o ControlMaster=auto -o ControlPersist=60 -o ControlPath=<job-local>` to
|
||||
reuse one connection instead of re-authenticating each time.
|
||||
|
||||
When to use what
|
||||
----------------
|
||||
|
||||
**Default: agent workers.** A Windmill *agent worker* is a lightweight worker
|
||||
that runs *inside* the target environment and connects back to the Windmill
|
||||
server over **outbound HTTP only** (using an `jwt_agent_*` token) — no inbound
|
||||
ports, no DB access. It keeps everything Windmill workers normally give you:
|
||||
automatic dependency management, nsjail sandboxing, the S3 binary cache, native
|
||||
secrets, all languages, and no per-job connection overhead. If you can run a
|
||||
process in the target environment, use an agent worker.
|
||||
|
||||
**Worker-group tags** are the right tool when you *can* place a full worker in
|
||||
the environment and want to route specific scripts to it.
|
||||
|
||||
**This SSH wrapper** is for the narrow case where **both** are true:
|
||||
|
||||
- you can only reach a **jump/utility node** over SSH (you cannot place any
|
||||
worker or agent process there), and
|
||||
- the scripts are **simple and self-contained** (no Windmill-managed deps).
|
||||
|
||||
What you lose with the SSH path
|
||||
-------------------------------
|
||||
|
||||
- **No dependency management.** The remote host must already have the interpreter
|
||||
*and* every library/tool the script uses. Nothing is installed or locked.
|
||||
- **No nsjail sandboxing.** The script runs as the SSH user with that user's full
|
||||
privileges. The jump node becomes a high-value target — scope the key and user
|
||||
tightly.
|
||||
- **No S3 / binary cache.** No shared cache of dependencies or artifacts.
|
||||
- **Per-job SSH overhead.** Each run pays connection + auth latency (mitigable
|
||||
with ControlMaster only if you make multiple round-trips).
|
||||
- **No native Windmill integrations on the remote side** — no resource/variable
|
||||
injection, no `wmill` client, no flow step context beyond what you pass in.
|
||||
|
||||
Limitations of this prototype
|
||||
-----------------------------
|
||||
|
||||
- Requires an `ssh` client (and `jq` for the bash variant) on the worker.
|
||||
- Assumes a self-contained, non-interactive script. No stdin is forwarded to the
|
||||
remote script (stdin carries the script body).
|
||||
- Unknown `language` values are passed through verbatim as the remote
|
||||
interpreter — keep `language` author-controlled, not end-user input.
|
||||
|
||||
Tested
|
||||
------
|
||||
|
||||
Both wrappers were exercised against a local `sshd`: success path, remote
|
||||
exit-code propagation (bash `${PIPESTATUS[1]}`, python raises), clean
|
||||
stdout/stderr separation, `python -u` interpreter dispatch, host-key pinning
|
||||
rejecting a wrong key (script never runs), the TOFU opt-in
|
||||
(`accept_unknown_host: true`) and the refusal without it, and confirmed remote
|
||||
*and* local temp-file cleanup.
|
||||
@@ -0,0 +1,116 @@
|
||||
# Windmill SSH execution wrapper (python)
|
||||
# --------------------------------------------------------------------------
|
||||
# Python variant of ssh_exec.sh. Same contract: run a self-contained script on
|
||||
# a remote host over SSH, stream stdout/stderr live, and fail the Windmill job
|
||||
# when the remote script fails. Userland prototype, no backend changes.
|
||||
#
|
||||
# Arguments:
|
||||
# ssh_target resource of type `ssh_target` (received as a dict)
|
||||
# script_content the body of the script to run on the remote host
|
||||
# language interpreter key: bash|sh|python|node|ruby|php|perl
|
||||
# (default: bash; anything else is treated as a raw remote
|
||||
# interpreter command)
|
||||
#
|
||||
# Worker requirements: an `ssh` client installed on the worker.
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
# `python3 -u` forces unbuffered output so logs stream live.
|
||||
INTERPRETERS = {
|
||||
"bash": "bash",
|
||||
"sh": "sh",
|
||||
"python": "python3 -u",
|
||||
"python3": "python3 -u",
|
||||
"node": "node",
|
||||
"javascript": "node",
|
||||
"ruby": "ruby",
|
||||
"php": "php",
|
||||
"perl": "perl",
|
||||
}
|
||||
|
||||
# Single-quoted (raw) so $f / $? / $TMPDIR are evaluated remotely, not here.
|
||||
# @@INTERP@@ is replaced with the chosen interpreter before sending.
|
||||
REMOTE_BOOTSTRAP = (
|
||||
"set -u\n"
|
||||
'f=$(mktemp "${TMPDIR:-/tmp}/wmssh_job.XXXXXX") || exit 1\n'
|
||||
"trap 'rm -f \"$f\"' EXIT\n" # remote-side cleanup, survives script failure
|
||||
'cat >"$f"\n' # read the streamed script body from stdin
|
||||
'@@INTERP@@ "$f"\n' # execute with the chosen interpreter
|
||||
"exit $?\n" # propagate the remote exit code
|
||||
)
|
||||
|
||||
|
||||
def main(ssh_target: dict, script_content: str, language: str = "bash"):
|
||||
host = ssh_target["host"]
|
||||
user = ssh_target["user"]
|
||||
private_key = ssh_target["private_key"]
|
||||
port = str(ssh_target.get("port") or 22)
|
||||
host_pubkey = (ssh_target.get("host_pubkey") or "").strip()
|
||||
accept_unknown_host = bool(ssh_target.get("accept_unknown_host"))
|
||||
|
||||
interp = INTERPRETERS.get(language, language) # passthrough for unknown keys
|
||||
|
||||
# 0600 temp files for the key and a job-local known_hosts.
|
||||
keyfile = tempfile.NamedTemporaryFile("w", delete=False)
|
||||
known_hosts = tempfile.NamedTemporaryFile("w", delete=False)
|
||||
try:
|
||||
keyfile.write(private_key.rstrip("\n") + "\n") # trailing newline required by some keys
|
||||
keyfile.close()
|
||||
os.chmod(keyfile.name, 0o600)
|
||||
|
||||
ssh_opts = [
|
||||
"-o", "BatchMode=yes",
|
||||
"-o", "ConnectTimeout=15",
|
||||
"-o", f"UserKnownHostsFile={known_hosts.name}",
|
||||
"-p", port,
|
||||
"-i", keyfile.name,
|
||||
]
|
||||
|
||||
if host_pubkey:
|
||||
# Pin the server key; non-default ports use the [host]:port form.
|
||||
entry = f"{host} {host_pubkey}" if port == "22" else f"[{host}]:{port} {host_pubkey}"
|
||||
known_hosts.write(entry + "\n")
|
||||
known_hosts.close()
|
||||
ssh_opts += ["-o", "StrictHostKeyChecking=yes"]
|
||||
elif accept_unknown_host:
|
||||
known_hosts.close()
|
||||
print(
|
||||
"WARN: ssh_target.host_pubkey is empty; using TOFU (accept-new) "
|
||||
"because accept_unknown_host=true. Pin host_pubkey for production.",
|
||||
file=sys.stderr,
|
||||
flush=True,
|
||||
)
|
||||
ssh_opts += ["-o", "StrictHostKeyChecking=accept-new"]
|
||||
else:
|
||||
known_hosts.close()
|
||||
raise ValueError(
|
||||
"ssh_target.host_pubkey is empty. Pin the host key "
|
||||
f"(ssh-keyscan -t ed25519 {host}) or set accept_unknown_host=true "
|
||||
"to allow TOFU (insecure against MITM)."
|
||||
)
|
||||
|
||||
remote = REMOTE_BOOTSTRAP.replace("@@INTERP@@", interp)
|
||||
# no -t/-tt: stdout and stderr stay separate for clean log capture.
|
||||
# `--` so a crafted user/host (e.g. "-oProxyCommand=...") can never be
|
||||
# parsed as an ssh option
|
||||
cmd = ["ssh", *ssh_opts, "--", f"{user}@{host}", remote]
|
||||
|
||||
# stdout/stderr are inherited from this process so they stream live to
|
||||
# the Windmill job log; only stdin is a pipe for the script body.
|
||||
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
|
||||
proc.communicate(input=(script_content + "\n").encode())
|
||||
rc = proc.returncode
|
||||
finally:
|
||||
for path in (keyfile.name, known_hosts.name):
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
# Raise on non-zero so the Windmill job fails with the remote exit code.
|
||||
if rc != 0:
|
||||
raise RuntimeError(f"Remote script exited with code {rc}")
|
||||
return {"ok": True, "exit_code": 0, "host": host}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Windmill SSH execution wrapper (bash)
|
||||
# --------------------------------------------------------------------------
|
||||
# Runs a self-contained script on a remote host reachable over SSH from the
|
||||
# Windmill worker, streaming stdout/stderr back live and propagating the
|
||||
# REMOTE exit code so a failed remote script fails the Windmill job.
|
||||
#
|
||||
# This is a userland prototype: it makes NO backend changes. See README.md for
|
||||
# when to use it (only-a-jump-node reachable, self-contained scripts) and why
|
||||
# agent workers are the recommended default.
|
||||
#
|
||||
# Windmill positional bash arguments:
|
||||
# 1. ssh_target resource of type `ssh_target` (Windmill passes it as JSON)
|
||||
# 2. script_content the body of the script to run on the remote host
|
||||
# 3. language interpreter key: bash|sh|python|node|ruby|php|perl
|
||||
# (default: bash; anything else is treated as a raw remote
|
||||
# interpreter command)
|
||||
#
|
||||
# Worker requirements: `ssh` client and `jq` must be installed on the worker.
|
||||
|
||||
ssh_target="$1"
|
||||
script_content="$2"
|
||||
language="${3:-bash}"
|
||||
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
command -v jq >/dev/null || { echo "FATAL: jq is required on the worker" >&2; exit 127; }
|
||||
command -v ssh >/dev/null || { echo "FATAL: ssh client is required on the worker" >&2; exit 127; }
|
||||
|
||||
# --- parse the ssh_target resource -----------------------------------------
|
||||
# `jq -e` exits non-zero when the field is null/absent, so a missing required
|
||||
# field fails fast with a clear message.
|
||||
host=$(jq -er '.host' <<<"$ssh_target") || { echo "FATAL: ssh_target.host missing" >&2; exit 2; }
|
||||
user=$(jq -er '.user' <<<"$ssh_target") || { echo "FATAL: ssh_target.user missing" >&2; exit 2; }
|
||||
private_key=$(jq -er '.private_key' <<<"$ssh_target") || { echo "FATAL: ssh_target.private_key missing" >&2; exit 2; }
|
||||
port=$(jq -r '.port // 22' <<<"$ssh_target")
|
||||
host_pubkey=$(jq -r '.host_pubkey // empty' <<<"$ssh_target")
|
||||
accept_unknown_host=$(jq -r '.accept_unknown_host // false' <<<"$ssh_target")
|
||||
|
||||
# --- interpreter dispatch table --------------------------------------------
|
||||
# `python3 -u` forces unbuffered output so logs stream live. For chatty bash
|
||||
# scripts that buffer, wrap the remote interpreter with `stdbuf -oL` (see README).
|
||||
case "$language" in
|
||||
bash) interp="bash" ;;
|
||||
sh) interp="sh" ;;
|
||||
python|python3) interp="python3 -u" ;;
|
||||
node|javascript) interp="node" ;;
|
||||
ruby) interp="ruby" ;;
|
||||
php) interp="php" ;;
|
||||
perl) interp="perl" ;;
|
||||
*) interp="$language" ;; # passthrough: raw remote interpreter command
|
||||
esac
|
||||
|
||||
# --- private key -> 0600 temp file + job-local known_hosts ------------------
|
||||
keyfile=$(mktemp "${TMPDIR:-/tmp}/wmssh_key.XXXXXX")
|
||||
known_hosts=$(mktemp "${TMPDIR:-/tmp}/wmssh_kh.XXXXXX")
|
||||
cleanup() { rm -f "$keyfile" "$known_hosts"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
printf '%s\n' "$private_key" >"$keyfile" # trailing newline: some keys require it
|
||||
chmod 600 "$keyfile"
|
||||
|
||||
# --- host-key handling ------------------------------------------------------
|
||||
ssh_opts=(
|
||||
-o BatchMode=yes
|
||||
-o ConnectTimeout=15
|
||||
-o "UserKnownHostsFile=$known_hosts"
|
||||
-p "$port"
|
||||
-i "$keyfile"
|
||||
)
|
||||
if [ -n "$host_pubkey" ]; then
|
||||
# Pin the server key: non-default ports use the `[host]:port` known_hosts form.
|
||||
if [ "$port" = "22" ]; then
|
||||
printf '%s %s\n' "$host" "$host_pubkey" >"$known_hosts"
|
||||
else
|
||||
printf '[%s]:%s %s\n' "$host" "$port" "$host_pubkey" >"$known_hosts"
|
||||
fi
|
||||
ssh_opts+=( -o StrictHostKeyChecking=yes )
|
||||
elif [ "$accept_unknown_host" = "true" ]; then
|
||||
echo "WARN: ssh_target.host_pubkey is empty; using TOFU (accept-new) because accept_unknown_host=true. Pin host_pubkey for production." >&2
|
||||
ssh_opts+=( -o StrictHostKeyChecking=accept-new )
|
||||
else
|
||||
echo "FATAL: ssh_target.host_pubkey is empty. Pin the host key (ssh-keyscan -t ed25519 $host) or set accept_unknown_host=true to allow TOFU (insecure against MITM)." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# --- remote bootstrap (runs ON the remote host) ----------------------------
|
||||
# Single-quoted heredoc: NOTHING is expanded locally. $f / $? / $TMPDIR are
|
||||
# all evaluated remotely. The script body arrives on the remote's stdin.
|
||||
remote_bootstrap=$(cat <<'REMOTE'
|
||||
set -u
|
||||
f=$(mktemp "${TMPDIR:-/tmp}/wmssh_job.XXXXXX") || exit 1
|
||||
trap 'rm -f "$f"' EXIT # remote-side cleanup, survives script failure
|
||||
cat >"$f" # read the streamed script body from stdin
|
||||
@@INTERP@@ "$f" # execute with the chosen interpreter
|
||||
exit $? # propagate the remote exit code
|
||||
REMOTE
|
||||
)
|
||||
# NOTE: `interp` is only controlled for the known dispatch keys above — the
|
||||
# `*)` case passes `language` through verbatim into the remote bootstrap, so
|
||||
# keep `language` author-controlled, never end-user input (see README).
|
||||
remote_bootstrap=${remote_bootstrap//@@INTERP@@/$interp}
|
||||
|
||||
# --- execute ---------------------------------------------------------------
|
||||
# - no -t/-tt: keep stdout and stderr separate for clean log capture
|
||||
# (add `-tt` ONLY when you need an interactive TTY, e.g. sudo password prompts)
|
||||
# - body is streamed via stdin so it never touches a local temp file
|
||||
# - PIPESTATUS[1] is ssh's exit code == the remote script's exit code
|
||||
set +e
|
||||
printf '%s\n' "$script_content" | ssh "${ssh_opts[@]}" -- "$user@$host" "$remote_bootstrap"
|
||||
rc=${PIPESTATUS[1]}
|
||||
exit "$rc"
|
||||
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"schema": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"host": {
|
||||
"type": "string",
|
||||
"description": "Hostname or IP of the SSH jump/utility node to execute on."
|
||||
},
|
||||
"port": {
|
||||
"type": "integer",
|
||||
"description": "SSH port.",
|
||||
"default": 22
|
||||
},
|
||||
"user": {
|
||||
"type": "string",
|
||||
"description": "SSH login user."
|
||||
},
|
||||
"private_key": {
|
||||
"type": "string",
|
||||
"description": "PEM-encoded private key used for authentication. Stored as a secret.",
|
||||
"password": true
|
||||
},
|
||||
"host_pubkey": {
|
||||
"type": "string",
|
||||
"description": "Server host public key line for known_hosts pinning, e.g. 'ssh-ed25519 AAAAC3Nz...' (get it with `ssh-keyscan -t ed25519 <host>`). When set, the wrapper enforces StrictHostKeyChecking=yes. When empty, execution is refused unless accept_unknown_host is true.",
|
||||
"default": ""
|
||||
},
|
||||
"accept_unknown_host": {
|
||||
"type": "boolean",
|
||||
"description": "Allow connecting without a pinned host_pubkey by trusting the host key on first use (StrictHostKeyChecking=accept-new). Insecure against MITM — development only.",
|
||||
"default": false
|
||||
}
|
||||
},
|
||||
"required": ["host", "user", "private_key"]
|
||||
},
|
||||
"description": "Target host reachable over SSH for the lightweight SSH execution wrapper. See examples/usecase/ssh-execution-wrapper/README.md."
|
||||
}
|
||||
@@ -322,6 +322,16 @@ export const settings: Record<string, Setting[]> = {
|
||||
'{\n "auths": {\n "myregistry.example.com": {\n "auth": "BASE64(username:password)"\n }\n }\n}',
|
||||
storage: 'setting'
|
||||
},
|
||||
{
|
||||
label: 'SSH execution (#ssh)',
|
||||
key: 'ssh_execution_enabled',
|
||||
fieldType: 'boolean',
|
||||
description:
|
||||
'Allow bash scripts starting with a <code>#ssh <resource_path></code> directive to run on the remote host described by the referenced <code>ssh_target</code> resource instead of the worker. Off by default.',
|
||||
storage: 'setting',
|
||||
ee_only: '',
|
||||
hideInQuickSetup: true
|
||||
},
|
||||
{
|
||||
label: 'Default timeout',
|
||||
key: 'job_default_timeout',
|
||||
|
||||
Reference in New Issue
Block a user