diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index f4fbf5bb92..bfd726a238 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1292,16 +1292,16 @@ async fn lock_modules<'c>( })?; // If we have local lockfiles (and they are enabled) we will replace script content with lockfile and tell hander that it is raw_deps job - let (content, raw_deps) = raw_deps + let (content_for_capture, raw_deps) = raw_deps .as_ref() .and_then(|llfs| llfs.get(language.as_str())) .map(|lock| (lock.to_owned(), true)) - .unwrap_or((content, false)); + .unwrap_or((content.clone(), false)); let new_lock = capture_dependency_job( &job.id, &language, - &content, + &content_for_capture, mem_peak, canceled_by, job_dir, @@ -1323,10 +1323,11 @@ async fn lock_modules<'c>( // let lock = match new_lock { Ok(new_lock) => { - tx = dependency_map - .patch(relative_imports.clone(), e.id.clone(), tx) - .await?; - + if !raw_deps { + tx = dependency_map + .patch(relative_imports.clone(), e.id.clone(), tx) + .await?; + } if language == ScriptLang::Bun || language == ScriptLang::Bunnative { let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); if anns.native && language == ScriptLang::Bun { @@ -1335,6 +1336,7 @@ async fn lock_modules<'c>( language = ScriptLang::Bun; }; } + Some(new_lock) } Err(error) => { diff --git a/cli/src/commands/flow/flow.ts b/cli/src/commands/flow/flow.ts index b6a59f95e1..e16f2fbcb3 100644 --- a/cli/src/commands/flow/flow.ts +++ b/cli/src/commands/flow/flow.ts @@ -248,7 +248,7 @@ async function generateLocks( ); if (candidate) { hasAny = true; - log.info(colors.green(`+ ${candidate}`)); + log.info(colors.yellow.bold(`~ ${candidate}`)); } } diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index 25835dea3f..664bf8fe3e 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -20,8 +20,17 @@ import { } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; import { GlobalDeps, exts, findGlobalDeps } from "../commands/script/script.ts"; -import { FSFSElement, findCodebase, yamlOptions } from "../commands/sync/sync.ts"; -import { generateHash, readInlinePathSync, getHeaders, writeIfChanged } from "./utils.ts"; +import { + FSFSElement, + findCodebase, + yamlOptions, +} from "../commands/sync/sync.ts"; +import { + generateHash, + readInlinePathSync, + getHeaders, + writeIfChanged, +} from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { FlowFile } from "../commands/flow/flow.ts"; import { replaceInlineScripts } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts"; @@ -37,7 +46,7 @@ export class LockfileGenerationError extends Error { } } -export async function generateAllMetadata() { } +export async function generateAllMetadata() {} function findClosestRawReqs( lang: LanguageWithRawReqsSupport | undefined, @@ -77,7 +86,6 @@ async function generateFlowHash( [, reqs] = Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; } - // Embed lock into hash hashes[f.path] = await generateHash( (await f.getContentText()) + (reqs ?? "") @@ -191,12 +199,11 @@ export async function generateFlowLockInternal( flowValue.value.modules, {}, SEP, - opts.defaultTs, + opts.defaultTs ); - inlineScripts - .forEach((s) => { - writeIfChanged(Deno.cwd() + SEP + folder + SEP + s.path, s.content); - }); + inlineScripts.forEach((s) => { + writeIfChanged(Deno.cwd() + SEP + folder + SEP + s.path, s.content); + }); // Overwrite `flow.yaml` with the new lockfile references writeIfChanged( @@ -206,7 +213,7 @@ export async function generateFlowLockInternal( } hashes = await generateFlowHash(rawReqs, folder, opts.defaultTs); - + await clearGlobalLock(folder); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); } @@ -428,7 +435,7 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch { } + } catch {} metadataContent.lock = ""; } } catch (e) { @@ -512,7 +519,7 @@ export async function updateFlow( } catch (e) { try { responseText = await rawResponse.text(); - } catch { } + } catch {} throw new Error( `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` ); @@ -534,16 +541,24 @@ export async function inferSchema( }> { let inferedSchema: any; if (language === "python3") { - const { parse_python } = await import("../../wasm/py/windmill_parser_wasm.js"); + const { parse_python } = await import( + "../../wasm/py/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_python(content)); } else if (language === "nativets") { - const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js"); + const { parse_deno } = await import( + "../../wasm/ts/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "bun") { - const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js"); + const { parse_deno } = await import( + "../../wasm/ts/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "deno") { - const { parse_deno } = await import("../../wasm/ts/windmill_parser_wasm.js"); + const { parse_deno } = await import( + "../../wasm/ts/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_deno(content)); } else if (language === "go") { const { parse_go } = await import("../../wasm/go/windmill_parser_wasm.js"); @@ -595,7 +610,9 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "postgresql") { - const { parse_sql } = await import("../../wasm/regex/windmill_parser_wasm.js"); + const { parse_sql } = await import( + "../../wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_sql(content)); inferedSchema.args = [ { name: "database", typ: { resource: "postgresql" } }, @@ -616,7 +633,9 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "bash") { - const { parse_bash } = await import("../../wasm/regex/windmill_parser_wasm.js"); + const { parse_bash } = await import( + "../../wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_bash(content)); } else if (language === "powershell") { const { parse_powershell } = await import( @@ -624,10 +643,14 @@ export async function inferSchema( ); inferedSchema = JSON.parse(parse_powershell(content)); } else if (language === "php") { - const { parse_php } = await import("../../wasm/php/windmill_parser_wasm.js"); + const { parse_php } = await import( + "../../wasm/php/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_php(content)); } else if (language === "rust") { - const { parse_rust } = await import("../../wasm/rust/windmill_parser_wasm.js"); + const { parse_rust } = await import( + "../../wasm/rust/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_rust(content)); } else if (language === "csharp") { const { parse_csharp } = await import( @@ -643,12 +666,16 @@ export async function inferSchema( ); inferedSchema = JSON.parse(parse_ansible(content)); } else if (language === "java") { - const { parse_java } = await import("../../wasm/java/windmill_parser_wasm.js"); + const { parse_java } = await import( + "../../wasm/java/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_java(content)); } else if (language === "ruby") { - const { parse_ruby } = await import("../../wasm/ruby/windmill_parser_wasm.js"); + const { parse_ruby } = await import( + "../../wasm/ruby/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_ruby(content)); - // for related places search: ADD_NEW_LANG + // for related places search: ADD_NEW_LANG } else { throw new Error("Invalid language: " + language); } @@ -736,10 +763,10 @@ export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: | (GlobalOptions & { - path: string; - workspaceRemote: Workspace; - schemaOnly?: boolean; - }) + path: string; + workspaceRemote: Workspace; + schemaOnly?: boolean; + }) | undefined, globalDeps: GlobalDeps, codebases: SyncCodebase[] @@ -885,6 +912,32 @@ export async function generateScriptHash( ); } +export async function clearGlobalLock(path: string): Promise { + const conf = await readLockfile(); + if (!conf?.locks) { + conf.locks = {}; + } + const isV2 = conf?.version == "v2"; + + if (isV2) { + // Remove the specific v2 lock entry + const key = v2LockPath(path); + if (conf.locks) { + Object.keys(conf.locks).forEach((k) => { + if (conf.locks) { + if (k.startsWith(key)) { + delete conf.locks[k]; + } + } + }); + } + await Deno.writeTextFile( + WMILL_LOCKFILE, + yamlStringify(conf as Record, yamlOptions) + ); + } +} + export async function updateMetadataGlobalLock( path: string, hash: string, @@ -897,7 +950,7 @@ export async function updateMetadataGlobalLock( const isV2 = conf?.version == "v2"; if (isV2) { - conf.locks[v2LockPath(path, hash)] = hash; + conf.locks[v2LockPath(path, subpath)] = hash; } else { if (subpath) { let prev: any = conf.locks[path];