diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index 34969f8e5f..d855910edd 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -222,7 +222,6 @@ pub fn parse_assets_ansible(code: &str) -> String { return serde_json::to_string(&r).unwrap(); } else { return format!("err: {:?}", o.err().unwrap()); - return "Invalid".to_string(); } } diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index dee9da5745..5e3af4f831 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -399,18 +399,31 @@ fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result anyhow::Result> { +#[derive(Debug, Clone, Serialize)] +pub struct DelegateWithSSHAuth { + delegate_to_git_repo_details: Option, + git_ssh_identity: Vec, +} + +pub fn parse_delegate_to_git_repo(inner_content: &str) -> anyhow::Result { let docs = YamlLoader::load_from_str(inner_content) .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; + let mut git_ssh_identity: Vec = vec![]; + if let Yaml::Hash(doc) = &docs[0] { + if let Some(v) = doc.get(&Yaml::String("git_ssh_identity".to_string())) { + let _ = extract_ssh_identity(&v, &mut git_ssh_identity); + } if let Some(v) = doc.get(&Yaml::String("delegate_to_git_repo".to_string())) { - return Ok(extract_delegate_to_git_repo_details(v)); + return Ok(DelegateWithSSHAuth { + delegate_to_git_repo_details: extract_delegate_to_git_repo_details(v), + git_ssh_identity, + }); } } - return Ok(None); + + Ok(DelegateWithSSHAuth { delegate_to_git_repo_details: None, git_ssh_identity }) } pub fn parse_ansible_reqs( @@ -420,7 +433,6 @@ pub fn parse_ansible_reqs( let docs = YamlLoader::load_from_str(inner_content) .map_err(|e| anyhow!("Failed to parse yaml: {}", e))?; - let mut ret = AnsibleRequirements::default(); if let Yaml::Hash(doc) = &docs[0] { @@ -528,23 +540,9 @@ pub fn parse_ansible_reqs( } } Yaml::String(key) if key == "git_ssh_identity" => { - let Yaml::Array(indentities) = &value else { - return Err(anyhow!( - "git_ssh_identity expects an array of windmill variables (or secrets) containing ssh IDs" - )); - }; - - for r in indentities { - let Yaml::String(file_name) = r else { - return Err(anyhow!( - "Git ssh identity file must be a string path to a Windmill variable/secret" - )); - }; - - ret.git_ssh_identity.push(file_name.clone()); - } + extract_ssh_identity(&value, &mut ret.git_ssh_identity)?; } - Yaml::String(key) if key == "delegate_to_git_repo" => {} + Yaml::String(key) if key == "delegate_to_git_repo" => {} // Skip this because it was already parsed before Yaml::String(key) => logs.push_str(&format!("\nUnknown field `{}`. Ignoring", key)), _ => (), } @@ -560,6 +558,25 @@ pub fn parse_ansible_reqs( Ok((logs, Some(ret), out_str)) } +fn extract_ssh_identity(value: &Yaml, ret: &mut Vec) -> anyhow::Result<()> { + let Yaml::Array(indentities) = value else { + return Err(anyhow!( + "git_ssh_identity expects an array of windmill variables (or secrets) containing ssh IDs" + )); + }; + + for r in indentities { + let Yaml::String(file_name) = r else { + return Err(anyhow!( + "Git ssh identity file must be a string path to a Windmill variable/secret" + )); + }; + + ret.push(file_name.clone()); + } + Ok(()) +} + fn extract_delegate_to_git_repo_details(value: &Yaml) -> Option { if let Yaml::Hash(v) = value { if let Some(resource) = v diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index ea799547a1..c90c69ef82 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4201,6 +4201,10 @@ paths: parameters: - $ref: "#/components/parameters/WorkspaceId" - $ref: "#/components/parameters/Path" + - name: git_ssh_identity + in: query + schema: + type: string responses: "200": description: git commit hash diff --git a/backend/windmill-api/src/resources.rs b/backend/windmill-api/src/resources.rs index 9decd976f8..faf525ade0 100644 --- a/backend/windmill-api/src/resources.rs +++ b/backend/windmill-api/src/resources.rs @@ -13,6 +13,7 @@ use crate::{ users::{maybe_refresh_folders, require_owner_of_path, Tokened}, utils::{check_scopes, require_super_admin, BulkDeleteRequest}, var_resource_cache::{cache_resource, get_cached_resource}, + variables::get_value_internal, webhook_util::{WebhookMessage, WebhookShared}, }; use axum::{ @@ -34,12 +35,12 @@ use uuid::Uuid; use windmill_audit::audit_oss::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::{ - db::{UserDB, UserDbWithOptAuthed}, - error::{Error, JsonResult, Result}, + db::{UserDB, UserDbWithAuthed, UserDbWithOptAuthed}, + error::{self, Error, JsonResult, Result}, get_database_url, parse_postgres_url, utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, variables, - worker::CLOUD_HOSTED, + worker::{CLOUD_HOSTED, TMP_DIR}, workspaces::get_ducklake_instance_pg_catalog_password, }; @@ -1403,12 +1404,18 @@ struct GitCommitHashResponse { commit_hash: String, } +#[derive(Deserialize)] +struct GitCommitHashQuery { + git_ssh_identity: Option, +} + async fn get_git_commit_hash( authed: ApiAuthed, Extension(user_db): Extension, Extension(db): Extension, Tokened { token }: Tokened, Path((w_id, path)): Path<(String, StripPath)>, + Query(query): Query, ) -> JsonResult { let path = path.to_path(); @@ -1416,7 +1423,7 @@ async fn get_git_commit_hash( let git_repo_resource_value = get_resource_value_interpolated_internal( &authed, - Some(user_db), + Some(user_db.clone()), &db, &w_id, path, @@ -1434,12 +1441,115 @@ async fn get_git_commit_hash( None => return Err(Error::NotFound(format!("Resource {} not found", path)).into()), }; - let commit_hash = get_repo_latest_commit_hash(&git_resource).await?; + let identities: Vec = query + .git_ssh_identity + .map(|s| s.split(",").map(|s| s.to_string()).collect()) + .unwrap_or(vec![]); - Ok(Json(GitCommitHashResponse { commit_hash })) + let (git_ssh_cmd, filenames) = + get_git_ssh_cmd(&authed, &user_db, &db, &w_id, identities).await?; + + let commit_hash = get_repo_latest_commit_hash(&git_resource, git_ssh_cmd).await; + + delete_paths(&filenames).await; + + Ok(Json(GitCommitHashResponse { commit_hash: commit_hash? })) } -async fn get_repo_latest_commit_hash(git_resource: &GitRepositoryResource) -> Result { +async fn write_ssh_file( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + var_path: &str, +) -> std::result::Result { + let id_file_name = format!(".ssh_id_priv_{}", Uuid::new_v4()); + let loc = std::path::Path::new(TMP_DIR) + .join("ssh_ids") + .join(id_file_name); + + let userdb_authed = UserDbWithAuthed { db: user_db.clone(), authed: &authed.to_authed_ref() }; + let mut content = get_value_internal(&userdb_authed, db, w_id, var_path, authed, false) + .await + .map_err(|e| { + ( + error::Error::NotFound(format!( + "Variable {var_path} not found for git ssh identity: {e:#}" + )), + loc.clone(), + ) + })?; + content.push_str("\n"); + + if let Some(p) = &loc.parent() { + tokio::fs::create_dir_all(p) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + tokio::fs::write(&loc, content) + .await + .map_err(|e| (e.into(), loc.clone()))?; + + #[cfg(unix)] + { + let perm = std::os::unix::fs::PermissionsExt::from_mode(0o600); + tokio::fs::set_permissions(&loc, perm) + .await + .map_err(|e| (e.into(), loc.clone()))?; + } + + return Ok(loc); +} + +async fn delete_paths(paths: &Vec) { + for path in paths { + let _ = tokio::fs::remove_file(&path).await; + } +} + +async fn get_git_ssh_cmd( + authed: &ApiAuthed, + user_db: &UserDB, + db: &DB, + w_id: &str, + git_ssh_identity: Vec, +) -> error::Result<(Option, Vec)> { + if git_ssh_identity.len() > 5 { + return Err(error::Error::BadRequest( + "Too many ssh identities, try using at most 1".to_string(), + )); + } + if git_ssh_identity.len() == 0 { + return Ok((None, vec![])); + } + + let mut ssh_id_files = vec![]; + let mut file_paths = vec![]; + for var_path in git_ssh_identity.iter() { + match write_ssh_file(authed, user_db, db, w_id, &var_path).await { + Ok(loc) => { + ssh_id_files.push(format!( + " -i '{}'", + loc.to_string_lossy().replace('\'', r"'\''") + )); + file_paths.push(loc); + } + Err((e, loc)) => { + file_paths.push(loc); + delete_paths(&file_paths).await; + return Err(e); + } + } + } + + let git_ssh_cmd = format!("ssh -o StrictHostKeyChecking=no{}", ssh_id_files.join("")); + Ok((Some(git_ssh_cmd), file_paths)) +} + +async fn get_repo_latest_commit_hash( + git_resource: &GitRepositoryResource, + git_ssh_command: Option, +) -> Result { let mut git_cmd = Command::new("git"); let ref_spec = git_resource @@ -1449,6 +1559,9 @@ async fn get_repo_latest_commit_hash(git_resource: &GitRepositoryResource) -> Re .unwrap_or("HEAD"); git_cmd.args(["ls-remote", &git_resource.url, ref_spec]); + if let Some(git_ssh_command) = git_ssh_command { + git_cmd.env("GIT_SSH_COMMAND", git_ssh_command); + } git_cmd.stderr(Stdio::piped()); let output = git_cmd diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 4f95518603..0de6d52aaa 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -83,7 +83,7 @@ "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.538.0", - "windmill-parser-wasm-yaml": "1.558.1", + "windmill-parser-wasm-yaml": "1.561.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", @@ -13779,9 +13779,9 @@ "integrity": "sha512-hHhMIVIPhmsHx0lsNCGMoIa7cDBFlVWhhd9j/5yOIq2sxwqg5sl5juQIIJGvuwg5umdsD0ChlSm2/uES78DLYg==" }, "node_modules/windmill-parser-wasm-yaml": { - "version": "1.558.1", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.558.1.tgz", - "integrity": "sha512-KBaSekkFiLJP5GpeArctupHStfp9/aWpNeT9my8Cp+QJB2TziJSf3FQ4k9mxJcKbjUQ65vO0jBtdNDvrnYufqQ==" + "version": "1.561.0", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-yaml/-/windmill-parser-wasm-yaml-1.561.0.tgz", + "integrity": "sha512-UbyxsRxJ/QDE+RFjj8q6cMZqr57gxHXBM+W8VLXnQ8I79W5KI+FhKcNFraUpXzqQjalZJ3cVZXXr8C7cTlJ8IQ==" }, "node_modules/windmill-sql-datatype-parser-wasm": { "version": "1.512.0", diff --git a/frontend/package.json b/frontend/package.json index 3fda798c10..f5aebe9181 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -148,7 +148,7 @@ "windmill-parser-wasm-ruby": "1.526.1", "windmill-parser-wasm-rust": "1.558.1", "windmill-parser-wasm-ts": "1.538.0", - "windmill-parser-wasm-yaml": "1.558.1", + "windmill-parser-wasm-yaml": "1.561.0", "windmill-sql-datatype-parser-wasm": "1.512.0", "windmill-utils-internal": "^1.3.1", "xterm": "^5.3.0", diff --git a/frontend/src/lib/components/GitRepoResourcePicker.svelte b/frontend/src/lib/components/GitRepoResourcePicker.svelte index bc5676370f..225f4507bc 100644 --- a/frontend/src/lib/components/GitRepoResourcePicker.svelte +++ b/frontend/src/lib/components/GitRepoResourcePicker.svelte @@ -12,6 +12,7 @@ currentCommit?: string currentInventories?: string currentPlaybook?: string + gitSshIdentity?: string[] } let { @@ -19,7 +20,8 @@ currentResource = undefined, currentCommit = undefined, currentInventories = undefined, - currentPlaybook = undefined + currentPlaybook = undefined, + gitSshIdentity = undefined }: Props = $props() const dispatch = createEventDispatcher<{ @@ -120,7 +122,8 @@ try { const result = await ResourceService.getGitCommitHash({ workspace: $workspaceStore!, - path: selectedResource + path: selectedResource, + gitSshIdentity: gitSshIdentity?.join(",") }) commitHash = result.commit_hash } catch (err) { diff --git a/frontend/src/lib/components/GitRepoViewer.svelte b/frontend/src/lib/components/GitRepoViewer.svelte index 35ada1ee61..4924a89529 100644 --- a/frontend/src/lib/components/GitRepoViewer.svelte +++ b/frontend/src/lib/components/GitRepoViewer.svelte @@ -18,10 +18,11 @@ interface Props { gitRepoResourcePath: string + gitSshIdentity?: string[] commitHashInput?: string } - let { gitRepoResourcePath, commitHashInput = $bindable() }: Props = $props() + let { gitRepoResourcePath, gitSshIdentity, commitHashInput = $bindable() }: Props = $props() let commitHash = $derived(commitHashInput); @@ -31,7 +32,9 @@ const payload = { workspace: workspace, - resource_path: gitRepoResourcePath + resource_path: gitRepoResourcePath, + git_ssh_identity: gitSshIdentity, + commit: commitHash, } isLoadingRepoClone = true @@ -47,7 +50,6 @@ tryCode: async () => { const testResult = await JobService.getCompletedJob({ workspace, id: jobId }) jobSuccess = !!testResult.success - console.log("res", testResult) if (jobSuccess) { await JobService.getCompletedJobResult({ workspace, id: jobId }) } else { @@ -79,10 +81,10 @@ if (!commitHash) { isLoadingCommitHash = true error = null - const result = await ResourceService.getGitCommitHash({ workspace: $workspaceStore!, - path: gitRepoResourcePath + path: gitRepoResourcePath, + gitSshIdentity: gitSshIdentity?.join(",") }) commitHashInput = result.commit_hash diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index fb7743cad1..d82b859fef 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -163,13 +163,15 @@ inferAnsibleExecutionMode(code).then((v) => { if ( v !== undefined && - (v === null || - v.resource !== ansibleAlternativeExecutionMode?.resource || - v.playbook !== ansibleAlternativeExecutionMode?.playbook || - v.inventories_location !== ansibleAlternativeExecutionMode?.inventories_location || - v.commit !== ansibleAlternativeExecutionMode?.commit) + (v.delegate_to_git_repo_details === null || + v.delegate_to_git_repo_details.resource !== ansibleAlternativeExecutionMode?.resource || + v.delegate_to_git_repo_details.playbook !== ansibleAlternativeExecutionMode?.playbook || + v.delegate_to_git_repo_details.inventories_location !== ansibleAlternativeExecutionMode?.inventories_location || + v.delegate_to_git_repo_details.commit !== ansibleAlternativeExecutionMode?.commit || + v.git_ssh_identity !== ansibleGitSshIdentity) ) { - ansibleAlternativeExecutionMode = v + ansibleAlternativeExecutionMode = v.delegate_to_git_repo_details + ansibleGitSshIdentity = v.git_ssh_identity } }) } @@ -200,6 +202,7 @@ | null | undefined >() + let ansibleGitSshIdentity = $state([]) const url = new URL(window.location.toString()) let initialCollab = /true|1/i.test(url.searchParams.get('collab') ?? '0') @@ -612,7 +615,9 @@ @@ -917,6 +922,7 @@ currentCommit={commitHashForGitRepo || ansibleAlternativeExecutionMode?.commit} currentInventories={ansibleAlternativeExecutionMode?.inventories_location} currentPlaybook={ansibleAlternativeExecutionMode?.playbook} + gitSshIdentity={ansibleGitSshIdentity} on:selected={handleDelegateConfigUpdate} on:addInventories={handleAddInventories} /> diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 3a82035564..2a015e9c38 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -37,5 +37,6 @@ "slackReport": "hub/9084/slack", "discordReport": "hub/9085/discord", "smtpReport": "hub/9086/smtp", - "cloneRepoToS3forGitRepoViewer": "hub/19825/clone_repo_and_upload_to_instance_storage" + "cloneRepoToS3forGitRepoViewer_0": "hub/19825/clone_repo_and_upload_to_instance_storage", + "cloneRepoToS3forGitRepoViewer": "hub/19827/clone_repo_and_upload_to_instance_storage" } diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 3510d84bcc..6f1d6d7829 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -114,7 +114,7 @@ export async function inferAssets( return [] } -export async function inferAnsibleExecutionMode(code: string) { +export async function inferAnsibleExecutionMode(code: string): any { try { await initWasmYaml() return JSON.parse(parse_ansible_delegate(code))