diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index d810319f5b..1f407e8a0e 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -229,6 +229,7 @@ pub struct AnsibleRequirements { pub vault_password_file: Option, pub vault_id: Vec, pub git_repos: Vec, + pub git_ssh_identity_files: Vec, } impl Default for AnsibleRequirements { @@ -250,6 +251,7 @@ impl Default for AnsibleRequirements { vault_password_file: None, vault_id: vec![], git_repos: vec![], + git_ssh_identity_files: vec![], } } } @@ -310,25 +312,7 @@ pub fn parse_ansible_reqs( return Ok((logs, None, inner_content.to_string())); } - let opts = AnsiblePlaybookOptions { - verbosity: None, - forks: None, - timeout: None, - flush_cache: None, - force_handlers: None, - }; - let mut ret = AnsibleRequirements { - python_reqs: vec![], - collections: None, - file_resources: vec![], - inventories: vec![], - vars: vec![], - resources: vec![], - options: opts, - vault_password_file: None, - vault_id: vec![], - git_repos: vec![], - }; + let mut ret = AnsibleRequirements::default(); if let Yaml::Hash(doc) = &docs[0] { for (key, value) in doc { @@ -420,6 +404,23 @@ pub fn parse_ansible_reqs( ); } } + Yaml::String(key) if key == "git_ssh_identity_files" => { + let Yaml::Array(indentity_files) = &value else { + return Err(anyhow!( + "git_ssh_identity_files expects an array of identity file names" + )); + }; + + for r in indentity_files { + let Yaml::String(file_name) = r else { + return Err(anyhow!( + "Git ssh identity file must be a string defining a file name" + )); + }; + + ret.git_ssh_identity_files.push(file_name.clone()); + } + } Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)), _ => (), } diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index ff64e08ce2..aa5d5c775b 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -5,9 +5,10 @@ use std::{collections::HashMap, os::unix::fs::PermissionsExt, path::PathBuf, pro use std::{collections::HashMap, path::PathBuf, process::Stdio}; use anyhow::anyhow; +use futures::future::try_join_all; use itertools::Itertools; use serde::{Deserialize, Serialize}; -use serde_json::value::RawValue; +use serde_json::{json, value::RawValue}; use tokio::process::Command; use uuid::Uuid; use windmill_common::{ @@ -54,6 +55,7 @@ async fn clone_repo( canceled_by: &mut Option, w_id: &str, occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, ) -> error::Result { let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; @@ -64,6 +66,7 @@ async fn clone_repo( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .args(["clone", "--quiet"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -100,6 +103,7 @@ async fn clone_repo( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .arg("-C") .arg(&target_path) .args(["checkout", "--quiet", commit]) @@ -133,6 +137,7 @@ async fn clone_repo( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .arg("-C") .arg(&target_path) .args(["rev-parse", "HEAD"]) @@ -188,6 +193,7 @@ async fn clone_repo_without_history( canceled_by: &mut Option, w_id: &str, occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, ) -> error::Result<()> { let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; @@ -234,6 +240,7 @@ async fn clone_repo_without_history( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .arg("-C") .arg(&target_path) .args(vec!["remote", "add", "origin", &repo.url]) @@ -265,6 +272,7 @@ async fn clone_repo_without_history( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .arg("-C") .arg(&target_path) .args(vec!["fetch", "--depth=1", "--quiet", "origin", full_commit]) @@ -296,6 +304,7 @@ async fn clone_repo_without_history( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .arg("-C") .arg(&target_path) .args(["checkout", "--quiet", "FETCH_HEAD"]) @@ -408,6 +417,7 @@ pub async fn install_galaxy_collections( canceled_by: &mut Option, conn: &Connection, occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, ) -> anyhow::Result<()> { write_file(job_dir, "requirements.yml", collections_yml)?; @@ -426,7 +436,15 @@ pub async fn install_galaxy_collections( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .args(vec!["role", "install", "-r", "requirements.yml", "-p", "./roles"]) + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .args(vec![ + "role", + "install", + "-r", + "requirements.yml", + "-p", + "./roles", + ]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -455,6 +473,7 @@ pub async fn install_galaxy_collections( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) + .env("GIT_SSH_COMMAND", git_ssh_cmd) .args(vec![ "collection", "install", @@ -586,10 +605,15 @@ pub async fn get_role_locks(job_dir: &str) -> anyhow::Result<(HashMap anyhow::Result { +pub async fn get_git_repo_full_head_commit_hash( + repo: &GitRepo, + git_ssh_cmd: &str, +) -> anyhow::Result { let mut git_cmd = Command::new(GIT_PATH.as_str()); - git_cmd.args(["ls-remote", &repo.url, "HEAD"]); + git_cmd + .env("GIT_SSH_COMMAND", git_ssh_cmd) + .args(["ls-remote", &repo.url, "HEAD"]); let output = git_cmd.stderr(Stdio::piped()).output().await?; @@ -628,6 +652,7 @@ pub async fn get_git_repos_lock( canceled_by: &mut Option, w_id: &str, occupancy_metrics: &mut OccupancyMetrics, + git_ssh_cmd: &str, ) -> anyhow::Result> { let mut ret = HashMap::new(); @@ -645,13 +670,14 @@ pub async fn get_git_repos_lock( canceled_by, w_id, occupancy_metrics, + git_ssh_cmd, ) .await?, ); } else { ret.insert( repo.url.to_string(), - get_git_repo_full_head_commit_hash(repo).await?, + get_git_repo_full_head_commit_hash(repo, git_ssh_cmd).await?, ); } } @@ -688,6 +714,40 @@ remote_tmp={job_dir}/.ansible/tmp Ok(()) } +pub async fn get_git_ssh_cmd(reqs: &AnsibleRequirements, job_dir: &str, client: &AuthedClient) -> error::Result { + let ssh_id_files = try_join_all(reqs + .git_ssh_identity_files + .iter() + .enumerate() + .map(async |(i, var_path)| -> error::Result { + let id_file_name = format!(".ssh_id_priv_{}", i); + let loc = is_allowed_file_location(job_dir, &id_file_name)?; + + let content = client + .get_variable_value(var_path) + .await + .map_err(|e| { + error::Error::NotFound(format!("Variable {var_path} not found for git ssh id file: {e:#}")) + })?; + + let file = write_file(job_dir, &id_file_name, &content)?; + + #[cfg(unix)] + { + let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); + file.set_permissions(perm)?; + } + + Ok(format!( + " -i '{}'", + loc.to_string_lossy().replace('\'', r"'\''") + )) + })).await?; + + let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); + Ok(git_ssh_cmd) +} + pub async fn handle_ansible_job( requirements_o: Option<&String>, job_dir: &str, @@ -734,6 +794,11 @@ pub async fn handle_ansible_job( ) .await?; + let git_ssh_cmd = &match &reqs { + Some(r) => get_git_ssh_cmd(r, job_dir, client).await?, + None => "ssh".to_string(), + }; + let interpolated_args; if let Some(args) = &job.args { let mut args = args.0.clone(); @@ -818,6 +883,7 @@ pub async fn handle_ansible_job( canceled_by, &job.workspace_id, occupancy_metrics, + git_ssh_cmd, ) .await .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?; @@ -841,6 +907,7 @@ pub async fn handle_ansible_job( canceled_by, &job.workspace_id, occupancy_metrics, + git_ssh_cmd, ) .await .map_err(|e| anyhow!("Failed to clone git repo `{}`: {e}", repo.url))?; @@ -866,6 +933,7 @@ pub async fn handle_ansible_job( canceled_by, conn, occupancy_metrics, + git_ssh_cmd, ) .await?; } diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index b8e64503f5..0165b74279 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1820,9 +1820,15 @@ async fn ansible_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut OccupancyMetrics, + token: &str, + base_internal_url: &str, ) -> std::result::Result { - use crate::ansible_executor::{ - create_ansible_cfg, get_collection_locks, get_role_locks, install_galaxy_collections, + use crate::{ + ansible_executor::{ + create_ansible_cfg, get_collection_locks, get_git_ssh_cmd, get_role_locks, + install_galaxy_collections, + }, + AuthedClient, }; let python_lockfile = python_dep( @@ -1843,6 +1849,15 @@ async fn ansible_dep( let conn = &Connection::Sql(db.clone()); + let authed_client = AuthedClient { + base_internal_url: base_internal_url.to_string(), + token: token.to_string(), + workspace: w_id.to_string(), + force_client: None, + }; + + let git_ssh_cmd = get_git_ssh_cmd(&reqs, job_dir, &authed_client).await?; + let git_repos = get_git_repos_lock( &reqs.git_repos, job_dir, @@ -1853,6 +1868,7 @@ async fn ansible_dep( canceled_by, w_id, occupancy_metrics, + &git_ssh_cmd, ) .await?; @@ -1871,6 +1887,7 @@ async fn ansible_dep( canceled_by, conn, occupancy_metrics, + &git_ssh_cmd, ) .await?; @@ -1994,6 +2011,8 @@ async fn capture_dependency_job( w_id, worker_dir, occupancy_metrics, + token, + base_internal_url, ) .await }