From 400b8308fd8f3f184566ebe0c41cc2e0d0652a6f Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sat, 1 Nov 2025 08:52:11 +0000 Subject: [PATCH] flow generate locks improvements --- .../windmill-worker/src/worker_lockfiles.rs | 41 ++++++++++--------- cli/src/commands/dev/dev.ts | 6 +-- cli/src/utils/metadata.ts | 12 ++++-- .../src/inline-scripts/extractor.ts | 3 +- .../src/inline-scripts/replacer.ts | 9 ++-- 5 files changed, 39 insertions(+), 32 deletions(-) diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index e448a120b6..24d79eb921 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -833,7 +833,7 @@ pub async fn handle_flow_dependency_job( &nodes_to_relock, occupancy_metrics, skip_flow_update, - raw_deps, + &raw_deps, &mut dependency_map, ) .await?; @@ -1029,7 +1029,7 @@ async fn lock_flow_value<'c>( locks_to_reload: &Option>, occupancy_metrics: &mut OccupancyMetrics, skip_flow_update: bool, - raw_deps: Option>, + raw_deps: &Option>, dependency_map: &mut ScopedDependencyMap, ) -> Result<( FlowValue, @@ -1057,7 +1057,7 @@ async fn lock_flow_value<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, ) .await?; @@ -1086,7 +1086,7 @@ async fn lock_flow_value<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, ) .await?; @@ -1121,7 +1121,7 @@ async fn lock_flow_value<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, ) .await?; @@ -1157,7 +1157,7 @@ async fn lock_modules<'c>( locks_to_reload: &Option>, occupancy_metrics: &mut OccupancyMetrics, skip_flow_update: bool, - raw_deps: Option>, + raw_deps: &Option>, dependency_map: &mut ScopedDependencyMap, // (modules to replace old seq (even unmmodified ones), new transaction, modified ids) ) ) -> Result<( Vec, @@ -1211,7 +1211,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, )) .await?; @@ -1247,7 +1247,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, )) .await?; @@ -1276,7 +1276,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, )) .await?; @@ -1309,7 +1309,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, )) .await?; @@ -1337,7 +1337,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, )) .await?; @@ -1405,7 +1405,7 @@ async fn lock_modules<'c>( locks_to_reload, occupancy_metrics, skip_flow_update, - raw_deps.clone(), + &raw_deps, dependency_map, )) .await?; @@ -1439,12 +1439,14 @@ async fn lock_modules<'c>( .await?; } - let dep_path = path.clone().unwrap_or_else(|| job_path.to_string()); - let relative_imports = extract_relative_imports( - &content, - &format!("{dep_path}/flow"), - &Some(language.clone()), - ); + let get_imports = || { + let dep_path = path.clone().unwrap_or_else(|| job_path.to_string()); + extract_relative_imports( + &content, + &format!("{dep_path}/flow"), + &Some(language.clone()), + ) + }; if let Some(locks_to_reload) = locks_to_reload { if !locks_to_reload.contains(&e.id) { @@ -1513,7 +1515,8 @@ async fn lock_modules<'c>( // let lock = match new_lock { Ok(new_lock) => { - if !raw_deps { + if !raw_deps && !skip_flow_update { + let relative_imports = get_imports(); tx = dependency_map .patch(relative_imports.clone(), e.id.clone(), tx) .await?; diff --git a/cli/src/commands/dev/dev.ts b/cli/src/commands/dev/dev.ts index 8eede6c359..87ef3458d1 100644 --- a/cli/src/commands/dev/dev.ts +++ b/cli/src/commands/dev/dev.ts @@ -19,7 +19,7 @@ import { mergeConfigWithConfigFile, readConfigFile, } from "../../core/conf.ts"; -import { exts, findGlobalDeps, removeExtensionToPath } from "../script/script.ts"; +import { exts, removeExtensionToPath } from "../script/script.ts"; import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { OpenFlow } from "../../../gen/types.gen.ts"; import { FlowFile } from "../flow/flow.ts"; @@ -82,8 +82,8 @@ async function dev(opts: GlobalOptions & SyncOptions) { localPath, SEP, undefined, - (path: string, newPath: string) => Deno.renameSync(path, newPath), - (path: string) => Deno.removeSync(path), + // (path: string, newPath: string) => Deno.renameSync(path, newPath), + // (path: string) => Deno.removeSync(path), ); currentLastEdit = { type: "flow", diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index bb94e2478e..00e2206933 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -183,8 +183,8 @@ export async function generateFlowLockInternal( folder + SEP!, SEP, changedScripts, - (path: string, newPath: string) => Deno.renameSync(path, newPath), - (path: string) => Deno.removeSync(path) + // (path: string, newPath: string) => Deno.renameSync(path, newPath), + // (path: string) => Deno.removeSync(path) ); //removeChangedLocks @@ -432,7 +432,9 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch {} + } catch (e) { + log.info(colors.yellow(`Error removing lock file ${lockPath}: ${e}`)); + } metadataContent.lock = ""; } } catch (e) { @@ -516,7 +518,9 @@ export async function updateFlow( } catch (e) { try { responseText = await rawResponse.text(); - } catch {} + } catch { + responseText = ""; + } throw new Error( `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` ); diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index b61cf60fb5..0239301ccb 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -34,8 +34,7 @@ export function extractInlineScripts( return modules.flatMap((m) => { if (m.value.type == "rawscript") { - let basePath, ext; - [basePath, ext] = assigner.assignPath(m.summary, m.value.language); + const [basePath, ext] = assigner.assignPath(m.summary, m.value.language); const path = mapping[m.id] ?? basePath + ext; const content = m.value.content; const r = [{ path: path, content: content }]; diff --git a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts index 9ec25e9f65..cad496a269 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/replacer.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/replacer.ts @@ -24,8 +24,8 @@ export async function replaceInlineScripts( localPath: string, separator: string = "/", removeLocks?: string[], - renamer?: (path: string, newPath: string) => void, - deleter?: (path: string) => void + // renamer?: (path: string, newPath: string) => void, + // deleter?: (path: string) => void ): Promise { await Promise.all(modules.map(async (module) => { if (!module.value) { @@ -34,7 +34,7 @@ export async function replaceInlineScripts( if (module.value.type === "rawscript" && module.value.content && module.value.content.startsWith("!inline")) { const path = module.value.content.split(" ")[1]; - const pathPrefix = path.split(".")[0]; + // const pathPrefix = path.split(".")[0]; const pathSuffix = path.split(".").slice(1).join("."); // new path is the module id with the same suffix const newPath = module.id + "." + pathSuffix; @@ -88,7 +88,8 @@ export async function replaceInlineScripts( try { module.value.lock = await fileReader(path.replaceAll("/", separator)); } catch { - logger.error(`Lock file ${path} not found`); + logger.error(`Lock file ${path} not found, treating as empty`); + module.value.lock = ""; } } } else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") {