diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index e958624f16..8ea219c54a 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -208,6 +208,15 @@ pub struct AnsibleInventory { resource_type: Option, pub pinned_resource: Option, } + +#[derive(Debug, Clone)] +pub struct GitRepo { + pub url: String, + pub commit: Option, + pub branch: Option, + pub target_path: String, +} + #[derive(Debug, Clone)] pub struct AnsibleRequirements { pub python_reqs: Vec, @@ -219,6 +228,7 @@ pub struct AnsibleRequirements { pub options: AnsiblePlaybookOptions, pub vault_password_file: Option, pub vault_id: Vec, + pub git_repos: Vec, } fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result> { @@ -294,6 +304,7 @@ pub fn parse_ansible_reqs( options: opts, vault_password_file: None, vault_id: vec![], + git_repos: vec![], }; if let Yaml::Hash(doc) = &docs[0] { @@ -351,7 +362,9 @@ pub fn parse_ansible_reqs( } Yaml::String(key) if key == "vault_password_file" => { let Yaml::String(filename) = value else { - return Err(anyhow!("Vault Password File expects a String containing the file name")); + return Err(anyhow!( + "Vault Password File expects a String containing the file name" + )); }; ret.vault_password_file = Some(filename.to_string()); } @@ -372,6 +385,18 @@ pub fn parse_ansible_reqs( ret.options = parse_ansible_options(opts); } } + Yaml::String(key) if key == "git_repos" => { + let Yaml::Array(repos) = &value else { + return Err(anyhow!("git_repos field expects an array of repos")); + }; + + for r in repos { + ret.git_repos.push( + parse_git_repo(r) + .map_err(|e| anyhow!("Failed to parse git repo: {e}"))?, + ); + } + } Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)), _ => (), } @@ -386,6 +411,38 @@ pub fn parse_ansible_reqs( Ok((logs, Some(ret), out_str)) } +fn parse_git_repo(r: &Yaml) -> anyhow::Result { + let Yaml::Hash(repo) = r else { + return Err(anyhow!("Should be a Map")); + }; + + let url = repo + .get(&Yaml::String("url".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .ok_or(anyhow!("Expected `url` field"))?; + + let target_path = repo + .get(&Yaml::String("target".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .ok_or(anyhow!( + "Expected `target` field (target directory for cloning the repo)" + ))?; + + let branch = repo + .get(&Yaml::String("branch".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + let commit = repo + .get(&Yaml::String("commit".to_string())) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + + Ok(GitRepo { url, commit, branch, target_path }) +} + fn parse_ansible_options(opts: &Vec) -> AnsiblePlaybookOptions { let mut ret = AnsiblePlaybookOptions { verbosity: None, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index a6979ad30e..d432ce6a49 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -251,11 +251,10 @@ fn normalize_path(path: &Path) -> PathBuf { } ret } -pub fn write_file_at_user_defined_location( + +pub fn is_allowed_file_location( job_dir: &str, user_defined_path: &str, - content: &str, - mode: Option, ) -> error::Result { let job_dir = Path::new(job_dir); let user_path = PathBuf::from(user_defined_path); @@ -275,6 +274,17 @@ pub fn write_file_at_user_defined_location( .into()); } + Ok(normalized_full_path) +} + +pub fn write_file_at_user_defined_location( + job_dir: &str, + user_defined_path: &str, + content: &str, + mode: Option, +) -> error::Result { + let normalized_full_path = is_allowed_file_location(job_dir, user_defined_path)?; + let full_path = normalized_full_path.as_path(); if let Some(parent_dir) = full_path.parent() { std::fs::create_dir_all(parent_dir)?; diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index bd69018428..99612fbf26 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -11,11 +11,11 @@ use tokio::process::Command; use uuid::Uuid; use windmill_common::{ error, - worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, + worker::{is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG}, }; use windmill_queue::MiniPulledJob; -use windmill_parser_yaml::{AnsibleRequirements, ResourceOrVariablePath}; +use windmill_parser_yaml::{AnsibleRequirements, GitRepo, ResourceOrVariablePath}; use windmill_queue::{append_logs, CanceledBy}; use crate::{ @@ -26,8 +26,8 @@ use crate::{ }, handle_child::handle_child, python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion}, - AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS, - PY_INSTALL_DIR, TZ_ENV, + AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV, + PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, }; lazy_static::lazy_static! { @@ -40,6 +40,84 @@ lazy_static::lazy_static! { const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto"); +async fn clone_repo( + repo: &GitRepo, + job_dir: &str, + job_id: &Uuid, + worker_name: &str, + db: &sqlx::Pool, + mem_peak: &mut i32, + canceled_by: &mut Option, + w_id: &str, + occupancy_metrics: &mut OccupancyMetrics, +) -> error::Result<()> { + let target_path = is_allowed_file_location(job_dir, &repo.target_path)?; + let mut clone_cmd = Command::new(GIT_PATH.as_str()); + clone_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .arg("clone") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(branch) = &repo.branch { + clone_cmd.args(["--branch", branch]); + } + clone_cmd.args([&repo.url, &target_path]); + + let clone_cmd_child = start_child_process(clone_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + clone_cmd_child, + false, + worker_name, + w_id, + "git clone", + None, + false, + &mut Some(occupancy_metrics), + ) + .await?; + + // Checkout specific commit if provided + if let Some(commit) = &repo.commit { + let mut checkout_cmd = Command::new(GIT_PATH.as_str()); + checkout_cmd + .current_dir(job_dir) + .env_clear() + .envs(PROXY_ENVS.clone()) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .args(["-C", &repo.target_path, "checkout", commit]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let checkout_cmd_child = start_child_process(checkout_cmd, GIT_PATH.as_str()).await?; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + checkout_cmd_child, + false, + worker_name, + w_id, + "git checkout", + None, + false, + &mut Some(occupancy_metrics), + ) + .await?; + } + + Ok(()) +} + async fn handle_ansible_python_deps( job_dir: &str, requirements_o: Option<&String>, @@ -144,13 +222,7 @@ async fn install_galaxy_collections( .envs(PROXY_ENVS.clone()) .env("PATH", PATH_ENV.as_str()) .env("TZ", TZ_ENV.as_str()) - .args(vec![ - "install", - "-r", - "requirements.yml", - "-p", - "./", - ]) + .args(vec!["install", "-r", "requirements.yml", "-p", "./"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -310,6 +382,21 @@ pub async fn handle_ansible_job( ) .await?; + for repo in r.git_repos { + clone_repo( + &repo, + job_dir, + &job.id, + worker_name, + db, + mem_peak, + canceled_by, + w_id, + occupancy_metrics, + ) + .await?; + } + if let Some(collections) = r.collections.as_ref() { install_galaxy_collections( collections, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index a1f1754ab5..f37df77c63 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -386,6 +386,7 @@ lazy_static::lazy_static! { pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string()); pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok();