flow generate locks improvements

backend/windmill-worker/src/worker_lockfiles.rs

@@ -833,7 +833,7 @@ pub async fn handle_flow_dependency_job(
         &nodes_to_relock,
         occupancy_metrics,
         skip_flow_update,
-        raw_deps,
+        &raw_deps,
         &mut dependency_map,
     )
     .await?;
@@ -1029,7 +1029,7 @@ async fn lock_flow_value<'c>(
     locks_to_reload: &Option<Vec<String>>,
     occupancy_metrics: &mut OccupancyMetrics,
     skip_flow_update: bool,
-    raw_deps: Option<HashMap<String, String>>,
+    raw_deps: &Option<HashMap<String, String>>,
     dependency_map: &mut ScopedDependencyMap,
 ) -> Result<(
     FlowValue,
@@ -1057,7 +1057,7 @@ async fn lock_flow_value<'c>(
         locks_to_reload,
         occupancy_metrics,
         skip_flow_update,
-        raw_deps.clone(),
+        &raw_deps,
         dependency_map,
     )
     .await?;
@@ -1086,7 +1086,7 @@ async fn lock_flow_value<'c>(
                 locks_to_reload,
                 occupancy_metrics,
                 skip_flow_update,
-                raw_deps.clone(),
+                &raw_deps,
                 dependency_map,
             )
             .await?;
@@ -1121,7 +1121,7 @@ async fn lock_flow_value<'c>(
             locks_to_reload,
             occupancy_metrics,
             skip_flow_update,
-            raw_deps.clone(),
+            &raw_deps,
             dependency_map,
         )
         .await?;
@@ -1157,7 +1157,7 @@ async fn lock_modules<'c>(
     locks_to_reload: &Option<Vec<String>>,
     occupancy_metrics: &mut OccupancyMetrics,
     skip_flow_update: bool,
-    raw_deps: Option<HashMap<String, String>>,
+    raw_deps: &Option<HashMap<String, String>>,
     dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) )
 ) -> Result<(
     Vec<FlowModule>,
@@ -1211,7 +1211,7 @@ async fn lock_modules<'c>(
                         locks_to_reload,
                         occupancy_metrics,
                         skip_flow_update,
-                        raw_deps.clone(),
+                        &raw_deps,
                         dependency_map,
                     ))
                     .await?;
@@ -1247,7 +1247,7 @@ async fn lock_modules<'c>(
                             locks_to_reload,
                             occupancy_metrics,
                             skip_flow_update,
-                            raw_deps.clone(),
+                            &raw_deps,
                             dependency_map,
                         ))
                         .await?;
@@ -1276,7 +1276,7 @@ async fn lock_modules<'c>(
                         locks_to_reload,
                         occupancy_metrics,
                         skip_flow_update,
-                        raw_deps.clone(),
+                        &raw_deps,
                         dependency_map,
                     ))
                     .await?;
@@ -1309,7 +1309,7 @@ async fn lock_modules<'c>(
                             locks_to_reload,
                             occupancy_metrics,
                             skip_flow_update,
-                            raw_deps.clone(),
+                            &raw_deps,
                             dependency_map,
                         ))
                         .await?;
@@ -1337,7 +1337,7 @@ async fn lock_modules<'c>(
                         locks_to_reload,
                         occupancy_metrics,
                         skip_flow_update,
-                        raw_deps.clone(),
+                        &raw_deps,
                         dependency_map,
                     ))
                     .await?;
@@ -1405,7 +1405,7 @@ async fn lock_modules<'c>(
                         locks_to_reload,
                         occupancy_metrics,
                         skip_flow_update,
-                        raw_deps.clone(),
+                        &raw_deps,
                         dependency_map,
                     ))
                     .await?;
@@ -1439,12 +1439,14 @@ async fn lock_modules<'c>(
             .await?;
         }
 
-        let dep_path = path.clone().unwrap_or_else(|| job_path.to_string());
-        let relative_imports = extract_relative_imports(
-            &content,
-            &format!("{dep_path}/flow"),
-            &Some(language.clone()),
-        );
+        let get_imports = || {
+            let dep_path = path.clone().unwrap_or_else(|| job_path.to_string());
+            extract_relative_imports(
+                &content,
+                &format!("{dep_path}/flow"),
+                &Some(language.clone()),
+            )
+        };
 
         if let Some(locks_to_reload) = locks_to_reload {
             if !locks_to_reload.contains(&e.id) {
@@ -1513,7 +1515,8 @@ async fn lock_modules<'c>(
         //
         let lock = match new_lock {
             Ok(new_lock) => {
-                if !raw_deps {
+                if !raw_deps && !skip_flow_update {
+                    let relative_imports = get_imports();
                     tx = dependency_map
                         .patch(relative_imports.clone(), e.id.clone(), tx)
                         .await?;

cli/src/commands/dev/dev.ts

@@ -19,7 +19,7 @@ import {
   mergeConfigWithConfigFile,
   readConfigFile,
 } from "../../core/conf.ts";
-import { exts, findGlobalDeps, removeExtensionToPath } from "../script/script.ts";
+import { exts, removeExtensionToPath } from "../script/script.ts";
 import { inferContentTypeFromFilePath } from "../../utils/script_common.ts";
 import { OpenFlow } from "../../../gen/types.gen.ts";
 import { FlowFile } from "../flow/flow.ts";
@@ -82,8 +82,8 @@ async function dev(opts: GlobalOptions & SyncOptions) {
           localPath,
           SEP,
           undefined,
-          (path: string, newPath: string) => Deno.renameSync(path, newPath),
-          (path: string) => Deno.removeSync(path),
+          // (path: string, newPath: string) => Deno.renameSync(path, newPath),
+          // (path: string) => Deno.removeSync(path),
         );
         currentLastEdit = {
           type: "flow",

cli/src/utils/metadata.ts

@@ -183,8 +183,8 @@ export async function generateFlowLockInternal(
       folder + SEP!,
       SEP,
       changedScripts,
-      (path: string, newPath: string) => Deno.renameSync(path, newPath),
-      (path: string) => Deno.removeSync(path)
+      // (path: string, newPath: string) => Deno.renameSync(path, newPath),
+      // (path: string) => Deno.removeSync(path)
     );
 
     //removeChangedLocks
@@ -432,7 +432,9 @@ async function updateScriptLock(
         if (await Deno.stat(lockPath)) {
           await Deno.remove(lockPath);
         }
-      } catch {}
+      } catch (e) {
+        log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`));
+      }
       metadataContent.lock = "";
     }
   } catch (e) {
@@ -516,7 +518,9 @@ export async function updateFlow(
   } catch (e) {
     try {
       responseText = await rawResponse.text();
-    } catch {}
+    } catch {
+      responseText = "";
+    }
     throw new Error(
       `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
     );

cli/windmill-utils-internal/src/inline-scripts/extractor.ts

@@ -34,8 +34,7 @@ export function extractInlineScripts(
 
   return modules.flatMap((m) => {
     if (m.value.type == "rawscript") {
-      let basePath, ext;
-      [basePath, ext] = assigner.assignPath(m.summary, m.value.language);
+      const [basePath, ext] = assigner.assignPath(m.summary, m.value.language);
       const path = mapping[m.id] ?? basePath + ext;
       const content = m.value.content;
       const r = [{ path: path, content: content }];

cli/windmill-utils-internal/src/inline-scripts/replacer.ts

@@ -24,8 +24,8 @@ export async function replaceInlineScripts(
     localPath: string,
     separator: string = "/",
     removeLocks?: string[],
-    renamer?: (path: string, newPath: string) => void,
-    deleter?: (path: string) => void
+    // renamer?: (path: string, newPath: string) => void,
+    // deleter?: (path: string) => void
   ): Promise<void> {
     await Promise.all(modules.map(async (module) => {
       if (!module.value) {
@@ -34,7 +34,7 @@ export async function replaceInlineScripts(
   
       if (module.value.type === "rawscript" && module.value.content && module.value.content.startsWith("!inline")) {
           const path = module.value.content.split(" ")[1];
-          const pathPrefix = path.split(".")[0];
+          // const pathPrefix = path.split(".")[0];
           const pathSuffix = path.split(".").slice(1).join(".");
           // new path is the module id with the same suffix
           const newPath = module.id + "." + pathSuffix;
@@ -88,7 +88,8 @@ export async function replaceInlineScripts(
             try {
               module.value.lock = await fileReader(path.replaceAll("/", separator));
             } catch {
-              logger.error(`Lock file ${path} not found`);
+              logger.error(`Lock file ${path} not found, treating as empty`);
+              module.value.lock = "";
             }
         }
       } else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") {