starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-07 08:02:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Clone arbitrary repos

Browse Source
This commit is contained in:
wendrul
2025-04-15 18:25:41 +02:00
parent a22d0e0680
commit 77c0f9b789
4 changed files with 170 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/parsers/windmill-parser-yaml/src/lib.rs
+58 -1
View File
@@ -208,6 +208,15 @@ pub struct AnsibleInventory {
resource_type: Option<String>,
pub pinned_resource: Option<String>,
}
#[derive(Debug, Clone)]
pub struct GitRepo {
pub url: String,
pub commit: Option<String>,
pub branch: Option<String>,
pub target_path: String,
}
#[derive(Debug, Clone)]
pub struct AnsibleRequirements {
pub python_reqs: Vec<String>,
@@ -219,6 +228,7 @@ pub struct AnsibleRequirements {
pub options: AnsiblePlaybookOptions,
pub vault_password_file: Option<String>,
pub vault_id: Vec<String>,
pub git_repos: Vec<GitRepo>,
}
fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInventory>> {
@@ -294,6 +304,7 @@ pub fn parse_ansible_reqs(
options: opts,
vault_password_file: None,
vault_id: vec![],
git_repos: vec![],
};
if let Yaml::Hash(doc) = &docs[0] {
@@ -351,7 +362,9 @@ pub fn parse_ansible_reqs(
}
Yaml::String(key) if key == "vault_password_file" => {
let Yaml::String(filename) = value else {
return Err(anyhow!("Vault Password File expects a String containing the file name"));
return Err(anyhow!(
"Vault Password File expects a String containing the file name"
));
};
ret.vault_password_file = Some(filename.to_string());
}
@@ -372,6 +385,18 @@ pub fn parse_ansible_reqs(
ret.options = parse_ansible_options(opts);
}
}
Yaml::String(key) if key == "git_repos" => {
let Yaml::Array(repos) = &value else {
return Err(anyhow!("git_repos field expects an array of repos"));
};
for r in repos {
ret.git_repos.push(
parse_git_repo(r)
.map_err(|e| anyhow!("Failed to parse git repo: {e}"))?,
);
}
}
Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)),
_ => (),
}
@@ -386,6 +411,38 @@ pub fn parse_ansible_reqs(
Ok((logs, Some(ret), out_str))
}
fn parse_git_repo(r: &Yaml) -> anyhow::Result<GitRepo> {
let Yaml::Hash(repo) = r else {
return Err(anyhow!("Should be a Map"));
};
let url = repo
.get(&Yaml::String("url".to_string()))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or(anyhow!("Expected `url` field"))?;
let target_path = repo
.get(&Yaml::String("target".to_string()))
.and_then(|v| v.as_str())
.map(|s| s.to_string())
.ok_or(anyhow!(
"Expected `target` field (target directory for cloning the repo)"
))?;
let branch = repo
.get(&Yaml::String("branch".to_string()))
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let commit = repo
.get(&Yaml::String("commit".to_string()))
.and_then(|v| v.as_str())
.map(|s| s.to_string());
Ok(GitRepo { url, commit, branch, target_path })
}
fn parse_ansible_options(opts: &Vec<Yaml>) -> AnsiblePlaybookOptions {
let mut ret = AnsiblePlaybookOptions {
verbosity: None,
backend/windmill-common/src/worker.rs
+13 -3
View File
@@ -251,11 +251,10 @@ fn normalize_path(path: &Path) -> PathBuf {
}
ret
}
pub fn write_file_at_user_defined_location(
pub fn is_allowed_file_location(
job_dir: &str,
user_defined_path: &str,
content: &str,
mode: Option<u32>,
) -> error::Result<PathBuf> {
let job_dir = Path::new(job_dir);
let user_path = PathBuf::from(user_defined_path);
@@ -275,6 +274,17 @@ pub fn write_file_at_user_defined_location(
.into());
}
Ok(normalized_full_path)
}
pub fn write_file_at_user_defined_location(
job_dir: &str,
user_defined_path: &str,
content: &str,
mode: Option<u32>,
) -> error::Result<PathBuf> {
let normalized_full_path = is_allowed_file_location(job_dir, user_defined_path)?;
let full_path = normalized_full_path.as_path();
if let Some(parent_dir) = full_path.parent() {
std::fs::create_dir_all(parent_dir)?;
backend/windmill-worker/src/ansible_executor.rs
+98 -11
View File
@@ -11,11 +11,11 @@ use tokio::process::Command;
use uuid::Uuid;
use windmill_common::{
error,
worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG},
worker::{is_allowed_file_location, to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG},
};
use windmill_queue::MiniPulledJob;
use windmill_parser_yaml::{AnsibleRequirements, ResourceOrVariablePath};
use windmill_parser_yaml::{AnsibleRequirements, GitRepo, ResourceOrVariablePath};
use windmill_queue::{append_logs, CanceledBy};
use crate::{
@@ -26,8 +26,8 @@ use crate::{
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile, PyVersion},
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, PROXY_ENVS,
PY_INSTALL_DIR, TZ_ENV,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, GIT_PATH, HOME_ENV, NSJAIL_PATH, PATH_ENV,
PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV,
};
lazy_static::lazy_static! {
@@ -40,6 +40,84 @@ lazy_static::lazy_static! {
const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto");
async fn clone_repo(
repo: &GitRepo,
job_dir: &str,
job_id: &Uuid,
worker_name: &str,
db: &sqlx::Pool<sqlx::Postgres>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
w_id: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<()> {
let target_path = is_allowed_file_location(job_dir, &repo.target_path)?;
let mut clone_cmd = Command::new(GIT_PATH.as_str());
clone_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.arg("clone")
.stdout(Stdio::piped())
.stderr(Stdio::piped());
if let Some(branch) = &repo.branch {
clone_cmd.args(["--branch", branch]);
}
clone_cmd.args([&repo.url, &target_path]);
let clone_cmd_child = start_child_process(clone_cmd, GIT_PATH.as_str()).await?;
handle_child(
job_id,
db,
mem_peak,
canceled_by,
clone_cmd_child,
false,
worker_name,
w_id,
"git clone",
None,
false,
&mut Some(occupancy_metrics),
)
.await?;
// Checkout specific commit if provided
if let Some(commit) = &repo.commit {
let mut checkout_cmd = Command::new(GIT_PATH.as_str());
checkout_cmd
.current_dir(job_dir)
.env_clear()
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.args(["-C", &repo.target_path, "checkout", commit])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let checkout_cmd_child = start_child_process(checkout_cmd, GIT_PATH.as_str()).await?;
handle_child(
job_id,
db,
mem_peak,
canceled_by,
checkout_cmd_child,
false,
worker_name,
w_id,
"git checkout",
None,
false,
&mut Some(occupancy_metrics),
)
.await?;
}
Ok(())
}
async fn handle_ansible_python_deps(
job_dir: &str,
requirements_o: Option<&String>,
@@ -144,13 +222,7 @@ async fn install_galaxy_collections(
.envs(PROXY_ENVS.clone())
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.args(vec![
"install",
"-r",
"requirements.yml",
"-p",
"./",
])
.args(vec!["install", "-r", "requirements.yml", "-p", "./"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
@@ -310,6 +382,21 @@ pub async fn handle_ansible_job(
)
.await?;
for repo in r.git_repos {
clone_repo(
&repo,
job_dir,
&job.id,
worker_name,
db,
mem_peak,
canceled_by,
w_id,
occupancy_metrics,
)
.await?;
}
if let Some(collections) = r.collections.as_ref() {
install_galaxy_collections(
collections,
backend/windmill-worker/src/worker.rs
+1
View File
@@ -386,6 +386,7 @@ lazy_static::lazy_static! {
pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string());
pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new());
pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
pub static ref GIT_PATH: String = std::env::var("GIT_PATH").unwrap_or_else(|_| "/usr/bin/git".to_string());
pub static ref NODE_PATH: Option<String> = std::env::var("NODE_PATH").ok();