feat(go): local go.mod (#5929)

* feat(go): local go.mod

* readability refactor

* remove dbg!

* ignore module

* remove space

Signed-off-by: pyranota <pyra@duck.com>

* Update backend/windmill-worker/src/go_executor.rs

---------

Signed-off-by: pyranota <pyra@duck.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/windmill-worker/src/go_executor.rs

@@ -3,7 +3,11 @@ use std::{collections::HashMap, fs::DirBuilder, process::Stdio};
 
 use itertools::Itertools;
 use serde_json::value::RawValue;
-use tokio::{fs::File, io::AsyncReadExt, process::Command};
+use tokio::{
+    fs::{self, File},
+    io::AsyncReadExt,
+    process::Command,
+};
 use uuid::Uuid;
 use windmill_common::{
     error::{self, Error},
@@ -91,6 +95,7 @@ pub async fn handle_go_job(
             true,
             skip_go_mod,
             skip_tidy,
+            false,
             worker_name,
             &job.workspace_id,
             occupation_metrics,
@@ -356,11 +361,21 @@ pub async fn install_go_dependencies(
     non_dep_job: bool,
     skip_go_mod: bool,
     has_sum: bool,
+    raw_deps: bool,
     worker_name: &str,
     w_id: &str,
     occupation_metrics: &mut OccupancyMetrics,
 ) -> error::Result<String> {
-    if !skip_go_mod {
+    if raw_deps {
+        let go_mod =
+            if let Some(module) = code.lines().find(|l| l.trim_start().starts_with("module ")) {
+                code.replace(module, "module mymod")
+            } else {
+                format!("module mymod\n{code}")
+            };
+        fs::write(format!("{job_dir}/go.mod"), go_mod).await?;
+    }
+    if !raw_deps && !skip_go_mod {
         gen_go_mymod(code, job_dir).await?;
         let mut child_cmd = Command::new(GO_PATH.as_str());
         child_cmd
@@ -400,7 +415,9 @@ pub async fn install_go_dependencies(
 
     let mut new_lockfile = false;
 
-    let hash = if !has_sum {
+    let hash = if raw_deps {
+        calculate_hash(code)
+    } else if !has_sum {
         calculate_hash(parse_go_imports(&code)?.iter().join("\n").as_str())
     } else {
         "".to_string()
@@ -429,7 +446,15 @@ pub async fn install_go_dependencies(
         }
     }
 
-    let mod_command = if skip_tidy { "download" } else { "tidy" };
+    let mod_command = if skip_tidy ||
+        // If there is go.mod provided we want to use `download` only.
+        // Unlike `tidy` it does not modify local go.mod
+        raw_deps
+    {
+        "download"
+    } else {
+        "tidy"
+    };
     let mut child_cmd = Command::new(GO_PATH.as_str());
     child_cmd
         .current_dir(job_dir)

backend/windmill-worker/src/worker_lockfiles.rs

@@ -2193,11 +2193,6 @@ async fn capture_dependency_job(
             }
         }
         ScriptLang::Go => {
-            if raw_deps {
-                return Err(Error::ExecutionErr(
-                    "Raw dependencies not supported for go".to_string(),
-                ));
-            }
             install_go_dependencies(
                 job_id,
                 job_raw_code,
@@ -2208,6 +2203,7 @@ async fn capture_dependency_job(
                 false,
                 false,
                 false,
+                raw_deps,
                 worker_name,
                 w_id,
                 occupancy_metrics,

cli/metadata.ts

@@ -40,7 +40,7 @@ export class LockfileGenerationError extends Error {
 export async function generateAllMetadata() {}
 
 function findClosestRawReqs(
-  lang: "bun" | "python3" | "php" | undefined,
+  lang: "bun" | "python3" | "php" | "go" | undefined,
   remotePath: string,
   globalDeps: GlobalDeps
 ): string | undefined {
@@ -72,6 +72,15 @@ function findClosestRawReqs(
         bestCandidate = { k, v };
       }
     });
+  } else if (lang == "go") {
+    Object.entries(globalDeps.goMods).forEach(([k, v]) => {
+      if (
+        remotePath.startsWith(k) &&
+        k.length >= (bestCandidate?.k ?? "").length
+      ) {
+        bestCandidate = { k, v };
+      }
+    });
   }
   // @ts-ignore
   return bestCandidate?.v;
@@ -193,14 +202,14 @@ export async function generateScriptMetadataInternal(
   const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
 
   const rawReqs = findClosestRawReqs(
-    language as "bun" | "python3" | "php" | undefined,
+    language as "bun" | "python3" | "php" | "go" | undefined,
     scriptPath,
     globalDeps
   );
   if (rawReqs) {
     log.info(
       (await blueColor())(
-        `Found raw requirements (package.json/requirements.txt/composer.json) for ${scriptPath}, using it`
+        `Found raw requirements (package.json/requirements.txt/composer.json/go.mod) for ${scriptPath}, using it`
       )
     );
   }

cli/script.ts

@@ -869,11 +869,13 @@ export type GlobalDeps = {
   pkgs: Record<string, string>;
   reqs: Record<string, string>;
   composers: Record<string, string>;
+  goMods: Record<string, string>;
 };
 export async function findGlobalDeps(): Promise<GlobalDeps> {
   const pkgs: { [key: string]: string } = {};
   const reqs: { [key: string]: string } = {};
   const composers: { [key: string]: string } = {};
+  const goMods: { [key: string]: string } = {};
   const els = await FSFSElement(Deno.cwd(), [], false);
   for await (const entry of readDirRecursiveWithIgnore((p, isDir) => {
     p = SEP + p;
@@ -882,21 +884,24 @@ export async function findGlobalDeps(): Promise<GlobalDeps> {
       !(
         p.endsWith(SEP + "package.json") ||
         p.endsWith(SEP + "requirements.txt") ||
-        p.endsWith(SEP + "composer.json")
+        p.endsWith(SEP + "composer.json") ||
+        p.endsWith(SEP + "go.mod")
       )
     );
   }, els)) {
     if (entry.isDirectory || entry.ignored) continue;
     const content = await entry.getContentText();
     if (entry.path.endsWith("package.json")) {
-      pkgs[entry.path.substring(0, entry.path.length - 12)] = content;
+      pkgs[entry.path.substring(0, entry.path.length - "package.json".length)] = content;
     } else if (entry.path.endsWith("requirements.txt")) {
-      reqs[entry.path.substring(0, entry.path.length - 16)] = content;
+      reqs[entry.path.substring(0, entry.path.length - "requirements.txt".length)] = content;
     } else if (entry.path.endsWith("composer.json")) {
-      composers[entry.path.substring(0, entry.path.length - 13)] = content;
+      composers[entry.path.substring(0, entry.path.length - "composer.json".length)] = content;
+    } else if (entry.path.endsWith("go.mod")) {
+      goMods[entry.path.substring(0, entry.path.length - "go.mod".length)] = content;
     }
   }
-  return { pkgs, reqs, composers };
+  return { pkgs, reqs, composers, goMods };
 }
 async function generateMetadata(
   opts: GlobalOptions & {