Simplify ansible ssh identity interface

backend/parsers/windmill-parser-yaml/src/lib.rs

@@ -229,6 +229,7 @@ pub struct AnsibleRequirements {
     pub vault_password_file: Option<String>,
     pub vault_id: Vec<String>,
     pub git_repos: Vec<GitRepo>,
+    pub git_ssh_identity_files: Vec<String>,
 }
 
 impl Default for AnsibleRequirements {
@@ -250,6 +251,7 @@ impl Default for AnsibleRequirements {
             vault_password_file: None,
             vault_id: vec![],
             git_repos: vec![],
+            git_ssh_identity_files: vec![],
         }
     }
 }
@@ -310,25 +312,7 @@ pub fn parse_ansible_reqs(
         return Ok((logs, None, inner_content.to_string()));
     }
 
-    let opts = AnsiblePlaybookOptions {
-        verbosity: None,
-        forks: None,
-        timeout: None,
-        flush_cache: None,
-        force_handlers: None,
-    };
-    let mut ret = AnsibleRequirements {
-        python_reqs: vec![],
-        collections: None,
-        file_resources: vec![],
-        inventories: vec![],
-        vars: vec![],
-        resources: vec![],
-        options: opts,
-        vault_password_file: None,
-        vault_id: vec![],
-        git_repos: vec![],
-    };
+    let mut ret = AnsibleRequirements::default();
 
     if let Yaml::Hash(doc) = &docs[0] {
         for (key, value) in doc {
@@ -420,6 +404,23 @@ pub fn parse_ansible_reqs(
                         );
                     }
                 }
+                Yaml::String(key) if key == "git_ssh_identity_files" => {
+                    let Yaml::Array(indentity_files) = &value else {
+                        return Err(anyhow!(
+                            "git_ssh_identity_files expects an array of identity file names"
+                        ));
+                    };
+
+                    for r in indentity_files {
+                        let Yaml::String(file_name) = r else {
+                            return Err(anyhow!(
+                                "Git ssh identity file must be a string defining a file name"
+                            ));
+                        };
+
+                        ret.git_ssh_identity_files.push(file_name.clone());
+                    }
+                }
                 Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)),
                 _ => (),
             }

backend/windmill-worker/src/ansible_executor.rs

@@ -5,9 +5,10 @@ use std::{collections::HashMap, os::unix::fs::PermissionsExt, path::PathBuf, pro
 use std::{collections::HashMap, path::PathBuf, process::Stdio};
 
 use anyhow::anyhow;
+use futures::future::try_join_all;
 use itertools::Itertools;
 use serde::{Deserialize, Serialize};
-use serde_json::value::RawValue;
+use serde_json::{json, value::RawValue};
 use tokio::process::Command;
 use uuid::Uuid;
 use windmill_common::{
@@ -54,6 +55,7 @@ async fn clone_repo(
     canceled_by: &mut Option<CanceledBy>,
     w_id: &str,
     occupancy_metrics: &mut OccupancyMetrics,
+    git_ssh_cmd: &str,
 ) -> error::Result<String> {
     let target_path = is_allowed_file_location(job_dir, &repo.target_path)?;
 
@@ -64,6 +66,7 @@ async fn clone_repo(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
         .args(["clone", "--quiet"])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
@@ -100,6 +103,7 @@ async fn clone_repo(
             .envs(PROXY_ENVS.clone())
             .env("PATH", PATH_ENV.as_str())
             .env("TZ", TZ_ENV.as_str())
+            .env("GIT_SSH_COMMAND", git_ssh_cmd)
             .arg("-C")
             .arg(&target_path)
             .args(["checkout", "--quiet", commit])
@@ -133,6 +137,7 @@ async fn clone_repo(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
         .arg("-C")
         .arg(&target_path)
         .args(["rev-parse", "HEAD"])
@@ -188,6 +193,7 @@ async fn clone_repo_without_history(
     canceled_by: &mut Option<CanceledBy>,
     w_id: &str,
     occupancy_metrics: &mut OccupancyMetrics,
+    git_ssh_cmd: &str,
 ) -> error::Result<()> {
     let target_path = is_allowed_file_location(job_dir, &repo.target_path)?;
 
@@ -234,6 +240,7 @@ async fn clone_repo_without_history(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
         .arg("-C")
         .arg(&target_path)
         .args(vec!["remote", "add", "origin", &repo.url])
@@ -265,6 +272,7 @@ async fn clone_repo_without_history(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
         .arg("-C")
         .arg(&target_path)
         .args(vec!["fetch", "--depth=1", "--quiet", "origin", full_commit])
@@ -296,6 +304,7 @@ async fn clone_repo_without_history(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
         .arg("-C")
         .arg(&target_path)
         .args(["checkout", "--quiet", "FETCH_HEAD"])
@@ -408,6 +417,7 @@ pub async fn install_galaxy_collections(
     canceled_by: &mut Option<CanceledBy>,
     conn: &Connection,
     occupancy_metrics: &mut OccupancyMetrics,
+    git_ssh_cmd: &str,
 ) -> anyhow::Result<()> {
     write_file(job_dir, "requirements.yml", collections_yml)?;
 
@@ -426,7 +436,15 @@ pub async fn install_galaxy_collections(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
-        .args(vec!["role", "install", "-r", "requirements.yml", "-p", "./roles"])
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
+        .args(vec![
+            "role",
+            "install",
+            "-r",
+            "requirements.yml",
+            "-p",
+            "./roles",
+        ])
         .stdout(Stdio::piped())
         .stderr(Stdio::piped());
 
@@ -455,6 +473,7 @@ pub async fn install_galaxy_collections(
         .envs(PROXY_ENVS.clone())
         .env("PATH", PATH_ENV.as_str())
         .env("TZ", TZ_ENV.as_str())
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
         .args(vec![
             "collection",
             "install",
@@ -586,10 +605,15 @@ pub async fn get_role_locks(job_dir: &str) -> anyhow::Result<(HashMap<String, St
     Ok((ret, logs))
 }
 
-pub async fn get_git_repo_full_head_commit_hash(repo: &GitRepo) -> anyhow::Result<String> {
+pub async fn get_git_repo_full_head_commit_hash(
+    repo: &GitRepo,
+    git_ssh_cmd: &str,
+) -> anyhow::Result<String> {
     let mut git_cmd = Command::new(GIT_PATH.as_str());
 
-    git_cmd.args(["ls-remote", &repo.url, "HEAD"]);
+    git_cmd
+        .env("GIT_SSH_COMMAND", git_ssh_cmd)
+        .args(["ls-remote", &repo.url, "HEAD"]);
 
     let output = git_cmd.stderr(Stdio::piped()).output().await?;
 
@@ -628,6 +652,7 @@ pub async fn get_git_repos_lock(
     canceled_by: &mut Option<CanceledBy>,
     w_id: &str,
     occupancy_metrics: &mut OccupancyMetrics,
+    git_ssh_cmd: &str,
 ) -> anyhow::Result<HashMap<String, String>> {
     let mut ret = HashMap::new();
 
@@ -645,13 +670,14 @@ pub async fn get_git_repos_lock(
                     canceled_by,
                     w_id,
                     occupancy_metrics,
+                    git_ssh_cmd,
                 )
                 .await?,
             );
         } else {
             ret.insert(
                 repo.url.to_string(),
-                get_git_repo_full_head_commit_hash(repo).await?,
+                get_git_repo_full_head_commit_hash(repo, git_ssh_cmd).await?,
             );
         }
     }
@@ -688,6 +714,40 @@ remote_tmp={job_dir}/.ansible/tmp
     Ok(())
 }
 
+pub async fn get_git_ssh_cmd(reqs: &AnsibleRequirements, job_dir: &str, client: &AuthedClient) -> error::Result<String> {
+    let ssh_id_files = try_join_all(reqs
+        .git_ssh_identity_files
+        .iter()
+        .enumerate()
+        .map(async |(i, var_path)| -> error::Result<String> {
+            let id_file_name = format!(".ssh_id_priv_{}", i);
+            let loc = is_allowed_file_location(job_dir, &id_file_name)?;
+
+            let content = client
+                .get_variable_value(var_path)
+                .await
+                .map_err(|e| {
+                    error::Error::NotFound(format!("Variable {var_path} not found for git ssh id file: {e:#}"))
+                })?;
+
+            let file = write_file(job_dir, &id_file_name, &content)?;
+
+            #[cfg(unix)]
+            {
+                let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600);
+                file.set_permissions(perm)?;
+            }
+
+            Ok(format!(
+                " -i '{}'",
+                loc.to_string_lossy().replace('\'', r"'\''")
+            ))
+        })).await?;
+
+    let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join(""));
+    Ok(git_ssh_cmd)
+}
+
 pub async fn handle_ansible_job(
     requirements_o: Option<&String>,
     job_dir: &str,
@@ -734,6 +794,11 @@ pub async fn handle_ansible_job(
     )
     .await?;
 
+    let git_ssh_cmd = &match &reqs {
+        Some(r) => get_git_ssh_cmd(r, job_dir, client).await?,
+        None => "ssh".to_string(),
+    };
+
     let interpolated_args;
     if let Some(args) = &job.args {
         let mut args = args.0.clone();
@@ -818,6 +883,7 @@ pub async fn handle_ansible_job(
                     canceled_by,
                     &job.workspace_id,
                     occupancy_metrics,
+                    git_ssh_cmd,
                 )
                 .await
                 .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?;
@@ -841,6 +907,7 @@ pub async fn handle_ansible_job(
                     canceled_by,
                     &job.workspace_id,
                     occupancy_metrics,
+                    git_ssh_cmd,
                 )
                 .await
                 .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?;
@@ -866,6 +933,7 @@ pub async fn handle_ansible_job(
                 canceled_by,
                 conn,
                 occupancy_metrics,
+                git_ssh_cmd,
             )
             .await?;
         }

backend/windmill-worker/src/worker_lockfiles.rs

@@ -1820,9 +1820,15 @@ async fn ansible_dep(
     w_id: &str,
     worker_dir: &str,
     occupancy_metrics: &mut OccupancyMetrics,
+    token: &str,
+    base_internal_url: &str,
 ) -> std::result::Result<String, Error> {
-    use crate::ansible_executor::{
-        create_ansible_cfg, get_collection_locks, get_role_locks, install_galaxy_collections,
+    use crate::{
+        ansible_executor::{
+            create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks,
+            install_galaxy_collections,
+        },
+        AuthedClient,
     };
 
     let python_lockfile = python_dep(
@@ -1843,6 +1849,15 @@ async fn ansible_dep(
 
     let conn = &Connection::Sql(db.clone());
 
+    let authed_client = AuthedClient {
+        base_internal_url: base_internal_url.to_string(),
+        token: token.to_string(),
+        workspace: w_id.to_string(),
+        force_client: None,
+    };
+
+    let git_ssh_cmd = get_git_ssh_cmd(&reqs, job_dir, &authed_client).await?;
+
     let git_repos = get_git_repos_lock(
         &reqs.git_repos,
         job_dir,
@@ -1853,6 +1868,7 @@ async fn ansible_dep(
         canceled_by,
         w_id,
         occupancy_metrics,
+        &git_ssh_cmd,
     )
     .await?;
 
@@ -1871,6 +1887,7 @@ async fn ansible_dep(
             canceled_by,
             conn,
             occupancy_metrics,
+            &git_ssh_cmd,
         )
         .await?;
 
@@ -1994,6 +2011,8 @@ async fn capture_dependency_job(
                     w_id,
                     worker_dir,
                     occupancy_metrics,
+                    token,
+                    base_internal_url,
                 )
                 .await
             }