fix(cli): improve generate-flow with raw requirements

backend/windmill-worker/src/worker_lockfiles.rs

@@ -1292,16 +1292,16 @@ async fn lock_modules<'c>(
         })?;
 
         // If we have local lockfiles (and they are enabled) we will replace script content with lockfile and tell hander that it is raw_deps job
-        let (content, raw_deps) = raw_deps
+        let (content_for_capture, raw_deps) = raw_deps
             .as_ref()
             .and_then(|llfs| llfs.get(language.as_str()))
             .map(|lock| (lock.to_owned(), true))
-            .unwrap_or((content, false));
+            .unwrap_or((content.clone(), false));
 
         let new_lock = capture_dependency_job(
             &job.id,
             &language,
-            &content,
+            &content_for_capture,
             mem_peak,
             canceled_by,
             job_dir,
@@ -1323,10 +1323,11 @@ async fn lock_modules<'c>(
         //
         let lock = match new_lock {
             Ok(new_lock) => {
-                tx = dependency_map
-                    .patch(relative_imports.clone(), e.id.clone(), tx)
-                    .await?;
-
+                if !raw_deps {
+                    tx = dependency_map
+                        .patch(relative_imports.clone(), e.id.clone(), tx)
+                        .await?;
+                }
                 if language == ScriptLang::Bun || language == ScriptLang::Bunnative {
                     let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
                     if anns.native && language == ScriptLang::Bun {
@@ -1335,6 +1336,7 @@ async fn lock_modules<'c>(
                         language = ScriptLang::Bun;
                     };
                 }
+
                 Some(new_lock)
             }
             Err(error) => {

cli/src/commands/flow/flow.ts

@@ -248,7 +248,7 @@ async function generateLocks(
       );
       if (candidate) {
         hasAny = true;
-        log.info(colors.green(`+ ${candidate}`));
+        log.info(colors.yellow.bold(`~ ${candidate}`));
       }
     }
 

cli/src/utils/metadata.ts

@@ -20,8 +20,17 @@ import {
 } from "./script_common.ts";
 import { inferContentTypeFromFilePath } from "./script_common.ts";
 import { GlobalDeps, exts, findGlobalDeps } from "../commands/script/script.ts";
-import { FSFSElement, findCodebase, yamlOptions } from "../commands/sync/sync.ts";
-import { generateHash, readInlinePathSync, getHeaders, writeIfChanged } from "./utils.ts";
+import {
+  FSFSElement,
+  findCodebase,
+  yamlOptions,
+} from "../commands/sync/sync.ts";
+import {
+  generateHash,
+  readInlinePathSync,
+  getHeaders,
+  writeIfChanged,
+} from "./utils.ts";
 import { SyncCodebase } from "./codebase.ts";
 import { FlowFile } from "../commands/flow/flow.ts";
 import { replaceInlineScripts } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts";
@@ -37,7 +46,7 @@ export class LockfileGenerationError extends Error {
   }
 }
 
-export async function generateAllMetadata() { }
+export async function generateAllMetadata() {}
 
 function findClosestRawReqs(
   lang: LanguageWithRawReqsSupport | undefined,
@@ -77,7 +86,6 @@ async function generateFlowHash(
         [, reqs] =
           Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? [];
       }
-
       // Embed lock into hash
       hashes[f.path] = await generateHash(
         (await f.getContentText()) + (reqs ?? "")
@@ -191,12 +199,11 @@ export async function generateFlowLockInternal(
       flowValue.value.modules,
       {},
       SEP,
-      opts.defaultTs,
+      opts.defaultTs
     );
-    inlineScripts
-      .forEach((s) => {
-        writeIfChanged(Deno.cwd() + SEP + folder + SEP + s.path, s.content);
-      });
+    inlineScripts.forEach((s) => {
+      writeIfChanged(Deno.cwd() + SEP + folder + SEP + s.path, s.content);
+    });
 
     // Overwrite `flow.yaml` with the new lockfile references
     writeIfChanged(
@@ -206,7 +213,7 @@ export async function generateFlowLockInternal(
   }
 
   hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs);
-
+  await clearGlobalLock(folder);
   for (const [path, hash] of Object.entries(hashes)) {
     await updateMetadataGlobalLock(folder, hash, path);
   }
@@ -428,7 +435,7 @@ async function updateScriptLock(
         if (await Deno.stat(lockPath)) {
           await Deno.remove(lockPath);
         }
-      } catch { }
+      } catch {}
       metadataContent.lock = "";
     }
   } catch (e) {
@@ -512,7 +519,7 @@ export async function updateFlow(
   } catch (e) {
     try {
       responseText = await rawResponse.text();
-    } catch { }
+    } catch {}
     throw new Error(
       `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
     );
@@ -534,16 +541,24 @@ export async function inferSchema(
 }> {
   let inferedSchema: any;
   if (language === "python3") {
-    const { parse_python } = await import("../../wasm/py/windmill_parser_wasm.js");
+    const { parse_python } = await import(
+      "../../wasm/py/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_python(content));
   } else if (language === "nativets") {
-    const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js");
+    const { parse_deno } = await import(
+      "../../wasm/ts/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_deno(content));
   } else if (language === "bun") {
-    const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js");
+    const { parse_deno } = await import(
+      "../../wasm/ts/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_deno(content));
   } else if (language === "deno") {
-    const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js");
+    const { parse_deno } = await import(
+      "../../wasm/ts/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_deno(content));
   } else if (language === "go") {
     const { parse_go } = await import("../../wasm/go/windmill_parser_wasm.js");
@@ -595,7 +610,9 @@ export async function inferSchema(
       ...inferedSchema.args,
     ];
   } else if (language === "postgresql") {
-    const { parse_sql } = await import("../../wasm/regex/windmill_parser_wasm.js");
+    const { parse_sql } = await import(
+      "../../wasm/regex/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_sql(content));
     inferedSchema.args = [
       { name: "database", typ: { resource: "postgresql" } },
@@ -616,7 +633,9 @@ export async function inferSchema(
       ...inferedSchema.args,
     ];
   } else if (language === "bash") {
-    const { parse_bash } = await import("../../wasm/regex/windmill_parser_wasm.js");
+    const { parse_bash } = await import(
+      "../../wasm/regex/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_bash(content));
   } else if (language === "powershell") {
     const { parse_powershell } = await import(
@@ -624,10 +643,14 @@ export async function inferSchema(
     );
     inferedSchema = JSON.parse(parse_powershell(content));
   } else if (language === "php") {
-    const { parse_php } = await import("../../wasm/php/windmill_parser_wasm.js");
+    const { parse_php } = await import(
+      "../../wasm/php/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_php(content));
   } else if (language === "rust") {
-    const { parse_rust } = await import("../../wasm/rust/windmill_parser_wasm.js");
+    const { parse_rust } = await import(
+      "../../wasm/rust/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_rust(content));
   } else if (language === "csharp") {
     const { parse_csharp } = await import(
@@ -643,12 +666,16 @@ export async function inferSchema(
     );
     inferedSchema = JSON.parse(parse_ansible(content));
   } else if (language === "java") {
-    const { parse_java } = await import("../../wasm/java/windmill_parser_wasm.js");
+    const { parse_java } = await import(
+      "../../wasm/java/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_java(content));
   } else if (language === "ruby") {
-    const { parse_ruby } = await import("../../wasm/ruby/windmill_parser_wasm.js");
+    const { parse_ruby } = await import(
+      "../../wasm/ruby/windmill_parser_wasm.js"
+    );
     inferedSchema = JSON.parse(parse_ruby(content));
-    // for related places search: ADD_NEW_LANG 
+    // for related places search: ADD_NEW_LANG
   } else {
     throw new Error("Invalid language: " + language);
   }
@@ -736,10 +763,10 @@ export async function parseMetadataFile(
   scriptPath: string,
   generateMetadataIfMissing:
     | (GlobalOptions & {
-      path: string;
-      workspaceRemote: Workspace;
-      schemaOnly?: boolean;
-    })
+        path: string;
+        workspaceRemote: Workspace;
+        schemaOnly?: boolean;
+      })
     | undefined,
   globalDeps: GlobalDeps,
   codebases: SyncCodebase[]
@@ -885,6 +912,32 @@ export async function generateScriptHash(
   );
 }
 
+export async function clearGlobalLock(path: string): Promise<void> {
+  const conf = await readLockfile();
+  if (!conf?.locks) {
+    conf.locks = {};
+  }
+  const isV2 = conf?.version == "v2";
+
+  if (isV2) {
+    // Remove the specific v2 lock entry
+    const key = v2LockPath(path);
+    if (conf.locks) {
+      Object.keys(conf.locks).forEach((k) => {
+        if (conf.locks) {
+          if (k.startsWith(key)) {
+            delete conf.locks[k];
+          }
+        }
+      });
+    }
+    await Deno.writeTextFile(
+      WMILL_LOCKFILE,
+      yamlStringify(conf as Record<string, any>, yamlOptions)
+    );
+  }
+}
+
 export async function updateMetadataGlobalLock(
   path: string,
   hash: string,
@@ -897,7 +950,7 @@ export async function updateMetadataGlobalLock(
   const isV2 = conf?.version == "v2";
 
   if (isV2) {
-    conf.locks[v2LockPath(path, hash)] = hash;
+    conf.locks[v2LockPath(path, subpath)] = hash;
   } else {
     if (subpath) {
       let prev: any = conf.locks[path];