From bb0ca5585599959b0720af1475c4b720f196d9ee Mon Sep 17 00:00:00 2001 From: pyranota <92104930+pyranota@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:17:31 +0200 Subject: [PATCH] feat(go): local go.mod (#5929) * feat(go): local go.mod * readability refactor * remove dbg! * ignore module * remove space Signed-off-by: pyranota * Update backend/windmill-worker/src/go_executor.rs --------- Signed-off-by: pyranota Co-authored-by: Ruben Fiszel --- backend/windmill-worker/src/go_executor.rs | 33 ++++++++++++++++--- .../windmill-worker/src/worker_lockfiles.rs | 6 +--- cli/metadata.ts | 15 +++++++-- cli/script.ts | 15 ++++++--- 4 files changed, 52 insertions(+), 17 deletions(-) diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 2217c775ba..c5fe55184a 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -3,7 +3,11 @@ use std::{collections::HashMap, fs::DirBuilder, process::Stdio}; use itertools::Itertools; use serde_json::value::RawValue; -use tokio::{fs::File, io::AsyncReadExt, process::Command}; +use tokio::{ + fs::{self, File}, + io::AsyncReadExt, + process::Command, +}; use uuid::Uuid; use windmill_common::{ error::{self, Error}, @@ -91,6 +95,7 @@ pub async fn handle_go_job( true, skip_go_mod, skip_tidy, + false, worker_name, &job.workspace_id, occupation_metrics, @@ -356,11 +361,21 @@ pub async fn install_go_dependencies( non_dep_job: bool, skip_go_mod: bool, has_sum: bool, + raw_deps: bool, worker_name: &str, w_id: &str, occupation_metrics: &mut OccupancyMetrics, ) -> error::Result { - if !skip_go_mod { + if raw_deps { + let go_mod = + if let Some(module) = code.lines().find(|l| l.trim_start().starts_with("module ")) { + code.replace(module, "module mymod") + } else { + format!("module mymod\n{code}") + }; + fs::write(format!("{job_dir}/go.mod"), go_mod).await?; + } + if !raw_deps && !skip_go_mod { gen_go_mymod(code, job_dir).await?; let mut child_cmd = Command::new(GO_PATH.as_str()); child_cmd @@ -400,7 +415,9 @@ pub async fn install_go_dependencies( let mut new_lockfile = false; - let hash = if !has_sum { + let hash = if raw_deps { + calculate_hash(code) + } else if !has_sum { calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str()) } else { "".to_string() @@ -429,7 +446,15 @@ pub async fn install_go_dependencies( } } - let mod_command = if skip_tidy { "download" } else { "tidy" }; + let mod_command = if skip_tidy || + // If there is go.mod provided we want to use `download` only. + // Unlike `tidy` it does not modify local go.mod + raw_deps + { + "download" + } else { + "tidy" + }; let mut child_cmd = Command::new(GO_PATH.as_str()); child_cmd .current_dir(job_dir) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 33c0375f09..47bb07c438 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -2193,11 +2193,6 @@ async fn capture_dependency_job( } } ScriptLang::Go => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for go".to_string(), - )); - } install_go_dependencies( job_id, job_raw_code, @@ -2208,6 +2203,7 @@ async fn capture_dependency_job( false, false, false, + raw_deps, worker_name, w_id, occupancy_metrics, diff --git a/cli/metadata.ts b/cli/metadata.ts index ee39d0b9da..d2d37aa22d 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -40,7 +40,7 @@ export class LockfileGenerationError extends Error { export async function generateAllMetadata() {} function findClosestRawReqs( - lang: "bun" | "python3" | "php" | undefined, + lang: "bun" | "python3" | "php" | "go" | undefined, remotePath: string, globalDeps: GlobalDeps ): string | undefined { @@ -72,6 +72,15 @@ function findClosestRawReqs( bestCandidate = { k, v }; } }); + } else if (lang == "go") { + Object.entries(globalDeps.goMods).forEach(([k, v]) => { + if ( + remotePath.startsWith(k) && + k.length >= (bestCandidate?.k ?? "").length + ) { + bestCandidate = { k, v }; + } + }); } // @ts-ignore return bestCandidate?.v; @@ -193,14 +202,14 @@ export async function generateScriptMetadataInternal( const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs); const rawReqs = findClosestRawReqs( - language as "bun" | "python3" | "php" | undefined, + language as "bun" | "python3" | "php" | "go" | undefined, scriptPath, globalDeps ); if (rawReqs) { log.info( (await blueColor())( - `Found raw requirements (package.json/requirements.txt/composer.json) for ${scriptPath}, using it` + `Found raw requirements (package.json/requirements.txt/composer.json/go.mod) for ${scriptPath}, using it` ) ); } diff --git a/cli/script.ts b/cli/script.ts index e12c15d9ba..87e4f85354 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -869,11 +869,13 @@ export type GlobalDeps = { pkgs: Record; reqs: Record; composers: Record; + goMods: Record; }; export async function findGlobalDeps(): Promise { const pkgs: { [key: string]: string } = {}; const reqs: { [key: string]: string } = {}; const composers: { [key: string]: string } = {}; + const goMods: { [key: string]: string } = {}; const els = await FSFSElement(Deno.cwd(), [], false); for await (const entry of readDirRecursiveWithIgnore((p, isDir) => { p = SEP + p; @@ -882,21 +884,24 @@ export async function findGlobalDeps(): Promise { !( p.endsWith(SEP + "package.json") || p.endsWith(SEP + "requirements.txt") || - p.endsWith(SEP + "composer.json") + p.endsWith(SEP + "composer.json") || + p.endsWith(SEP + "go.mod") ) ); }, els)) { if (entry.isDirectory || entry.ignored) continue; const content = await entry.getContentText(); if (entry.path.endsWith("package.json")) { - pkgs[entry.path.substring(0, entry.path.length - 12)] = content; + pkgs[entry.path.substring(0, entry.path.length - "package.json".length)] = content; } else if (entry.path.endsWith("requirements.txt")) { - reqs[entry.path.substring(0, entry.path.length - 16)] = content; + reqs[entry.path.substring(0, entry.path.length - "requirements.txt".length)] = content; } else if (entry.path.endsWith("composer.json")) { - composers[entry.path.substring(0, entry.path.length - 13)] = content; + composers[entry.path.substring(0, entry.path.length - "composer.json".length)] = content; + } else if (entry.path.endsWith("go.mod")) { + goMods[entry.path.substring(0, entry.path.length - "go.mod".length)] = content; } } - return { pkgs, reqs, composers }; + return { pkgs, reqs, composers, goMods }; } async function generateMetadata( opts: GlobalOptions & {