fix: various improvements for ansible (#4419)

* Improve error messages, fix dependency job issue

* Add nsjail support for ansible

* Remove invalid comment syntax

* Change nsjail config and ansible controller config to be ephemeral

* Move ansible cfg to later to overwrite possible files named ansible.cfg

* Fix ansible.cfg env var

* Fix tmp dir issue

backend/windmill-api/src/scripts.rs

@@ -583,6 +583,7 @@ async fn create_script_internal<'c>(
         || ns.language == ScriptLang::Bunnative
         || ns.language == ScriptLang::Deno
         || ns.language == ScriptLang::Rust
+        || ns.language == ScriptLang::Ansible
         || ns.language == ScriptLang::Php)
     {
         Some(String::new())

backend/windmill-common/src/worker.rs

@@ -198,7 +198,7 @@ pub fn write_file_at_user_defined_location(
     job_dir: &str,
     user_defined_path: &str,
     content: &str,
-) -> error::Result<File> {
+) -> error::Result<PathBuf> {
     let job_dir = Path::new(job_dir);
     let user_path = PathBuf::from(user_defined_path);
 
@@ -217,6 +217,7 @@ pub fn write_file_at_user_defined_location(
         .into());
     }
 
+    let full_path = normalized_full_path.as_path();
     if let Some(parent_dir) = full_path.parent() {
         std::fs::create_dir_all(parent_dir)?;
     }
@@ -224,7 +225,7 @@ pub fn write_file_at_user_defined_location(
     let mut file = File::create(full_path)?;
     file.write_all(content.as_bytes())?;
     file.flush()?;
-    Ok(file)
+    Ok(normalized_full_path)
 }
 
 pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {

backend/windmill-worker/nsjail/run.ansible.config.proto

@@ -0,0 +1,140 @@
+name: "ansible run script"
+
+mode: ONCE
+hostname: "ansible"
+log_level: ERROR
+
+rlimit_as: 4096
+rlimit_cpu: 1000
+rlimit_fsize: 1000
+rlimit_nofile: 10000
+
+cwd: "/tmp"
+
+clone_newnet: false
+clone_newuser: {CLONE_NEWUSER}
+
+keep_caps: false
+keep_env: true
+
+mount {
+    src: "/bin"
+    dst: "/bin"
+    is_bind: true
+}
+
+mount {
+    src: "/lib"
+    dst: "/lib"
+    is_bind: true
+}
+
+
+mount {
+    src: "/lib64"
+    dst: "/lib64"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "/usr"
+    dst: "/usr"
+    is_bind: true
+}
+
+mount {
+    src: "/dev/null"
+    dst: "/dev/null"
+    is_bind: true
+    rw: true
+}
+
+mount {
+    dst: "/dev/shm"
+    fstype: "tmpfs"
+    rw: true
+    is_bind: false
+}
+
+mount {
+    dst: "/tmp"
+    fstype: "tmpfs"
+    rw: true
+    options: "size=800000000"
+}
+
+mount {
+    src: "{JOB_DIR}/main.yml"
+    dst: "/tmp/main.yml"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/wrapper.sh"
+    dst: "/tmp/wrapper.sh"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/requirements.yml"
+    dst: "/tmp/requirements.yml"
+    is_bind: true
+    mandatory: false
+}
+
+mount {
+    src: "{JOB_DIR}/ansible.cfg"
+    dst: "/tmp/ansible.cfg"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/ansible_collections/"
+    dst: "/tmp/ansible_collections/"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/args.json"
+    dst: "/tmp/args.json"
+    is_bind: true
+}
+
+mount {
+    src: "{JOB_DIR}/result.json"
+    dst: "/tmp/result_nsjail_mount.json"
+    rw: true
+    is_bind: true
+}
+
+mount {
+    src: "/etc"
+    dst: "/etc"
+	is_bind: true
+}
+
+mount {
+    src: "/dev/random"
+    dst: "/dev/random"
+    is_bind: true
+}
+
+mount {
+    src: "/dev/urandom"
+    dst: "/dev/urandom"
+    is_bind: true
+}
+
+{SHARED_MOUNT}
+
+{SHARED_DEPENDENCIES}
+
+{FILE_RESOURCES}
+
+iface_no_lo: true
+
+envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
+envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}"
+envar: "HOME=/tmp"
+envar: "ANSIBLE_CONFIG=/tmp/ansible.cfg"

backend/windmill-worker/src/ansible_executor.rs

@@ -1,6 +1,9 @@
-use std::{collections::HashMap, process::Stdio};
+use std::{
+    collections::HashMap, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::Stdio
+};
 
 use anyhow::anyhow;
+use itertools::Itertools;
 use serde_json::value::RawValue;
 use tokio::process::Command;
 use uuid::Uuid;
@@ -13,12 +16,10 @@ use windmill_parser_yaml::AnsibleRequirements;
 use windmill_queue::{append_logs, CanceledBy};
 
 use crate::{
-    common::{
-        get_reserved_variables, handle_child, read_and_check_result,
-        start_child_process, transform_json,
-    },
-    python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
-    AuthedClientBackgroundTask, DISABLE_NSJAIL, HOME_ENV, PATH_ENV, TZ_ENV,
+    bash_executor::BIN_BASH, common::{
+        get_reserved_variables, handle_child, read_and_check_result, start_child_process,
+        transform_json,
+    }, python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV
 };
 
 lazy_static::lazy_static! {
@@ -29,6 +30,8 @@ lazy_static::lazy_static! {
     std::env::var("ANSIBLE_GALAXY_PATH").unwrap_or("/usr/local/bin/ansible-galaxy".to_string());
 }
 
+const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto");
+
 async fn handle_ansible_python_deps(
     job_dir: &str,
     requirements_o: Option<String>,
@@ -98,6 +101,7 @@ async fn handle_ansible_python_deps(
     }
     Ok(additional_python_paths)
 }
+
 async fn install_galaxy_collections(
     collections_yml: &str,
     job_dir: &str,
@@ -155,6 +159,24 @@ async fn install_galaxy_collections(
     Ok(())
 }
 
+#[cfg(not(feature = "enterprise"))]
+fn check_ansible_exists() -> Result<(), error::Error> {
+    if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
+        let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
+        return Err(error::Error::NotFound(msg));
+    }
+    Ok(())
+}
+
+#[cfg(feature = "enterprise")]
+fn check_ansible_exists() -> Result<(), error::Error> {
+    if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
+        let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
+        return Err(error::Error::NotFound(msg));
+    }
+    Ok(())
+}
+
 pub async fn handle_ansible_job(
     requirements_o: Option<String>,
     job_dir: &str,
@@ -166,19 +188,16 @@ pub async fn handle_ansible_job(
     db: &sqlx::Pool<sqlx::Postgres>,
     client: &AuthedClientBackgroundTask,
     inner_content: &String,
+    shared_mount: &str,
     base_internal_url: &str,
     envs: HashMap<String, String>,
 ) -> windmill_common::error::Result<Box<RawValue>> {
+    check_ansible_exists()?;
+
     let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?;
     append_logs(&job.id, &job.workspace_id, logs, db).await;
     write_file(job_dir, "main.yml", &playbook)?;
 
-    let ansible_cfg_content = r#"
-[defaults]
-collections_path = ./
-roles_path = ./roles
-"#;
-    write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
 
     let additional_python_paths = handle_ansible_python_deps(
         job_dir,
@@ -215,6 +234,7 @@ roles_path = ./roles
         interpolated_args = None;
         write_file(job_dir, "args.json", "{}")?;
     };
+    write_file(job_dir, "result.json", "")?;
 
     let inventories: Vec<String> = reqs
         .as_ref()
@@ -228,8 +248,9 @@ roles_path = ./roles
         .unwrap_or_else(|| vec![]);
 
     let authed_client = client.get_authed().await;
+    let mut nsjail_extra_mounts = vec![];
     if let Some(r) = reqs {
-        create_file_resources(
+        nsjail_extra_mounts = create_file_resources(
             &job.id,
             &job.workspace_id,
             job_dir,
@@ -261,18 +282,97 @@ roles_path = ./roles
         db,
     )
     .await;
-
-    if !*DISABLE_NSJAIL {
-        return Err(anyhow!("Ansible is not supported with nsjail, disable nsjail on your worker to run ansible playbooks").into());
-    }
+    let ansible_cfg_content = format!(r#"
+[defaults]
+collections_path = ./
+roles_path = ./roles
+home={job_dir}/.ansible
+local_tmp={job_dir}/.ansible/tmp
+remote_tmp={job_dir}/.ansible/tmp
+"#);
+    write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
 
     let mut reserved_variables = get_reserved_variables(job, &authed_client.token, db).await?;
     let additional_python_paths_folders = additional_python_paths.join(":");
-    reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
+
+    if !*DISABLE_NSJAIL {
+        let shared_deps = additional_python_paths
+            .into_iter()
+            .map(|pp| {
+                format!(
+                    r#"
+mount {{
+    src: "{pp}"
+    dst: "{pp}"
+    is_bind: true
+    rw: false
+}}
+        "#
+                )
+            })
+            .join("\n");
+        let _ = write_file(
+            job_dir,
+            "run.config.proto",
+            &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT
+                .replace("{JOB_DIR}", job_dir)
+                .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
+                .replace("{SHARED_MOUNT}", shared_mount)
+                .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
+                .replace("{FILE_RESOURCES}", nsjail_extra_mounts.join("\n").as_str())
+                .replace(
+                    "{ADDITIONAL_PYTHON_PATHS}",
+                    additional_python_paths_folders.as_str(),
+                ),
+        )?;
+    } else {
+        reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
+    }
 
     let mut cmd_args = vec!["main.yml", "--extra-vars", "@args.json"];
     cmd_args.extend(inventories.iter().map(|s| s.as_str()));
-    let child = {
+
+    let child = if !*DISABLE_NSJAIL {
+
+        let wrapper = format!(r#"set -eou pipefail
+{0} "$@"
+if [ -f "result" ]; then
+    cat result > result_nsjail_mount.json
+fi
+if [ -f "result.json" ]; then
+    cat result.json > result_nsjail_mount.json
+fi
+"#, ANSIBLE_PLAYBOOK_PATH.as_str());
+
+        let file = write_file(job_dir, "wrapper.sh", &wrapper)?;
+
+        file.metadata()?.permissions().set_mode(0o777);
+        // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
+        nsjail_cmd
+            .current_dir(job_dir)
+            .env_clear()
+            // inject PYTHONPATH here - for some reason I had to do it in nsjail conf
+            .envs(reserved_variables)
+            .env("PATH", PATH_ENV.as_str())
+            .env("TZ", TZ_ENV.as_str())
+            .env("BASE_INTERNAL_URL", base_internal_url)
+            .env("BASE_URL", base_internal_url)
+            .args(
+                vec![
+                    "--config",
+                    "run.config.proto",
+                    "--",
+                    BIN_BASH.as_str(),
+                    "/tmp/wrapper.sh",
+                ]
+                .into_iter()
+                .chain(cmd_args),
+            )
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped());
+        start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
+    } else {
         let mut ansible_cmd = Command::new(ANSIBLE_PLAYBOOK_PATH.as_str());
         ansible_cmd
             .current_dir(job_dir)
@@ -306,6 +406,24 @@ roles_path = ./roles
     read_and_check_result(job_dir).await
 }
 
+fn define_nsjail_mount(job_dir: &str, path: &PathBuf) -> anyhow::Result<String> {
+    Ok(format!(
+        r#"
+mount {{
+    src: "{0}/{1}"
+    dst: "/tmp/{1}"
+    is_bind: true
+    rw: false
+    mandatory: false
+}}
+        "#,
+        job_dir,
+        path.strip_prefix(job_dir)?
+            .to_str()
+            .ok_or(anyhow!("Invalid path."))?
+    ))
+}
+
 async fn create_file_resources(
     job_id: &Uuid,
     w_id: &str,
@@ -314,8 +432,9 @@ async fn create_file_resources(
     r: &AnsibleRequirements,
     client: &crate::AuthedClient,
     db: &sqlx::Pool<sqlx::Postgres>,
-) -> error::Result<()> {
+) -> error::Result<Vec<String>> {
     let mut logs = String::new();
+    let mut nsjail_mounts: Vec<String> = vec![];
 
     for inventory in &r.inventories {
         let content;
@@ -336,9 +455,13 @@ async fn create_file_resources(
 
             content = serde_json::from_str(o.get())
                 .map_err(|e| anyhow!("Failed to parse inventory arg: {}", e))?;
+
+            if content == serde_json::value::Value::Null {
+                Err(anyhow!("The inventory argument was left empty. If you do not wish to specify an inventory for this script, remove the `inventory:` section from the yaml."))?;
+            }
         }
 
-        write_file_at_user_defined_location(
+        let validated_path = write_file_at_user_defined_location(
             job_dir,
             &inventory.name,
             content
@@ -349,6 +472,12 @@ async fn create_file_resources(
                 ))?,
         )
         .map_err(|e| anyhow!("Couldn't write inventory: {}", e))?;
+
+        nsjail_mounts.push(
+            define_nsjail_mount(job_dir, &validated_path)
+                .map_err(|e| anyhow!("Inventory path (a.k.a. `name`) is invalid: {}", e))?,
+        );
+
         logs.push_str(&format!("\nCreated inventory `{}`", inventory.name));
     }
 
@@ -360,7 +489,7 @@ async fn create_file_resources(
             )
             .await?;
         let path = file_res.target_path.clone();
-        write_file_at_user_defined_location(
+        let validated_path = write_file_at_user_defined_location(
             job_dir,
             path.as_str(),
             r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!(
@@ -369,6 +498,12 @@ async fn create_file_resources(
             ))?,
         )
         .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
+
+        nsjail_mounts.push(
+            define_nsjail_mount(job_dir, &validated_path)
+                .map_err(|e| anyhow!("File resource path is invalid: {}", e))?,
+        );
+
         logs.push_str(&format!(
             "\nCreated {} from {}",
             file_res.target_path, file_res.resource_path
@@ -376,5 +511,5 @@ async fn create_file_resources(
     }
     append_logs(job_id, w_id, logs, db).await;
 
-    Ok(())
+    Ok(nsjail_mounts)
 }

backend/windmill-worker/src/bash_executor.rs

@@ -12,7 +12,7 @@ use windmill_common::{
 use windmill_queue::{append_logs, CanceledBy};
 
 lazy_static::lazy_static! {
-    static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string());
+    pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string());
 }
 const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
 const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str =

backend/windmill-worker/src/worker.rs

@@ -2849,6 +2849,7 @@ mount {{
                 db,
                 client,
                 &inner_content,
+                &shared_mount,
                 base_internal_url,
                 envs,
             ).await