perf(cli): skip relock more accurate (#7993)

* perf(cli): skip relock more accurate

Signed-off-by: pyranota <pyra@duck.com>

* Update cli/src/utils/metadata.ts

Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>

* Update cli/src/commands/flow/flow_metadata.ts

Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>

* fix

Signed-off-by: pyranota <pyra@duck.com>

* use structuredClone for safety

Signed-off-by: pyranota <pyra@duck.com>

* chore: update ee-repo-ref to 592848d59ca2304926fb2bd85d000668a7f46a77

This commit updates the EE repository reference after PR #420 was merged in windmill-ee-private.

Previous ee-repo-ref: 931813b75b8260faa13ddc07f36a11607b7e3bf6

New ee-repo-ref: 592848d59ca2304926fb2bd85d000668a7f46a77

Automated by sync-ee-ref workflow.

* fix ci

Signed-off-by: pyranota <pyra@duck.com>

* add simple tests

Signed-off-by: pyranota <pyra@duck.com>

---------

Signed-off-by: pyranota <pyra@duck.com>
Co-authored-by: claude[bot] <209825114+claude[bot]@users.noreply.github.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>

cli/src/commands/app/app_metadata.ts

@@ -16,6 +16,9 @@ import {
   inferSchema,
   getRawWorkspaceDependencies,
   normalizeLockPath,
+  filterWorkspaceDependencies,
+  filterWorkspaceDependenciesForScripts,
+  InlineScriptInfo,
 } from "../../utils/metadata.ts";
 import {
   ScriptLanguage,
@@ -118,8 +121,23 @@ export async function generateAppLocksInternal(
   const rawWorkspaceDependencies: Record<string, string> =
     await getRawWorkspaceDependencies();
 
-  let hashes = await generateAppHash(
+  // Read the app file first to filter workspace dependencies
+  const appFilePath = path.join(
+    appFolder,
+    rawApp ? "raw_app.yaml" : "app.yaml"
+  );
+  const appFile = (await yamlParseFile(appFilePath)) as AppFile;
+
+  // Filter workspace dependencies based on inline scripts' languages and annotations
+  const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value;
+  const filteredDeps = await filterWorkspaceDependenciesForApp(
+    appValue,
     rawWorkspaceDependencies,
+    appFolder
+  );
+
+  let hashes = await generateAppHash(
+    filteredDeps,
     appFolder,
     rawApp,
     opts.defaultTs
@@ -141,7 +159,7 @@ export async function generateAppLocksInternal(
     return remote_path;
   }
 
-  if (Object.keys(rawWorkspaceDependencies).length > 0) {
+  if (Object.keys(filteredDeps).length > 0) {
     log.info(
       (await blueColor())(
         `Found workspace dependencies (${workspaceDependenciesLanguages
@@ -151,13 +169,6 @@ export async function generateAppLocksInternal(
     );
   }
 
-  // Read the app file
-  const appFilePath = path.join(
-    appFolder,
-    rawApp ? "raw_app.yaml" : "app.yaml"
-  );
-  const appFile = (await yamlParseFile(appFilePath)) as AppFile;
-
   if (!justUpdateMetadataLock) {
     const changedScripts = [];
     // Find hashes that do not correspond to previous hashes
@@ -195,7 +206,7 @@ export async function generateAppLocksInternal(
           runnables,
           remote_path,
           appFolder,
-          rawWorkspaceDependencies,
+          filteredDeps,
           opts.defaultTs
         );
         // Note: updateRawAppRunnables now writes each runnable to its own file
@@ -211,7 +222,7 @@ export async function generateAppLocksInternal(
           normalAppFile.value,
           remote_path,
           appFolder,
-          rawWorkspaceDependencies,
+          filteredDeps,
           opts.defaultTs
         );
 
@@ -228,7 +239,7 @@ export async function generateAppLocksInternal(
 
   // Regenerate hashes after updates
   hashes = await generateAppHash(
-    rawWorkspaceDependencies,
+    filteredDeps,
     appFolder,
     rawApp,
     opts.defaultTs
@@ -252,6 +263,31 @@ type InlineScriptProcessor = (
   }
 ) => Promise<any>;
 
+/**
+ * Filters raw workspace dependencies for an app by traversing all inline scripts,
+ * filtering deps for each based on language and annotations, then computing the union.
+ */
+export async function filterWorkspaceDependenciesForApp(
+  appValue: any,
+  rawWorkspaceDependencies: Record<string, string>,
+  folder: string
+): Promise<Record<string, string>> {
+  // Collect all inline scripts (use clone to avoid any mutations)
+  const scripts: InlineScriptInfo[] = [];
+
+  await traverseAndProcessInlineScripts(structuredClone(appValue), async (inlineScript) => {
+    if (inlineScript.content && inlineScript.language) {
+      scripts.push({
+        content: inlineScript.content,
+        language: inlineScript.language as ScriptLanguage,
+      });
+    }
+    return inlineScript;
+  });
+
+  return await filterWorkspaceDependenciesForScripts(scripts, rawWorkspaceDependencies, folder, SEP);
+}
+
 /**
  * Traverses an app structure (either app.value for normal apps or app.runnables for raw apps)
  * and processes all inline scripts found, returning the updated structure
@@ -552,6 +588,11 @@ async function generateInlineScriptLock(
   scriptPath: string,
   rawWorkspaceDependencies: Record<string, string> | undefined
 ): Promise<string> {
+  // Filter workspace dependencies to only include those matching this script's language and annotations
+  const filteredDeps = rawWorkspaceDependencies
+    ? filterWorkspaceDependencies(rawWorkspaceDependencies, content, language as ScriptLanguage)
+    : undefined;
+
   const extraHeaders = getHeaders();
 
   const rawResponse = await fetch(
@@ -572,9 +613,8 @@ async function generateInlineScriptLock(
           },
         ],
         raw_workspace_dependencies:
-          rawWorkspaceDependencies &&
-          Object.keys(rawWorkspaceDependencies).length > 0
-            ? rawWorkspaceDependencies
+          filteredDeps && Object.keys(filteredDeps).length > 0
+            ? filteredDeps
             : null,
         entrypoint: scriptPath,
       }),

cli/src/commands/flow/flow_metadata.ts

@@ -16,7 +16,9 @@ import {
   LockfileGenerationError,
   getRawWorkspaceDependencies,
   normalizeLockPath,
+  filterWorkspaceDependenciesForScripts,
 } from "../../utils/metadata.ts";
+import { ScriptLanguage } from "../../utils/script_common.ts";
 import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
 
 import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
@@ -70,8 +72,16 @@ export async function generateFlowLockInternal(
   // Always get out-of-sync workspace dependencies
   const rawWorkspaceDependencies: Record<string, string> =
     await getRawWorkspaceDependencies();
+
+  const flowValue = (await yamlParseFile(
+    folder! + SEP + "flow.yaml"
+  )) as FlowFile;
+
+  // Filter workspace dependencies based on inline scripts' languages and annotations
+  const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
+
   let hashes = await generateFlowHash(
-    rawWorkspaceDependencies,
+    filteredDeps,
     folder,
     opts.defaultTs
   );
@@ -88,7 +98,7 @@ export async function generateFlowLockInternal(
     return remote_path;
   }
 
-  if (Object.keys(rawWorkspaceDependencies).length > 0) {
+  if (Object.keys(filteredDeps).length > 0) {
     log.info(
       (await blueColor())(
         `Found workspace dependencies (${workspaceDependenciesLanguages
@@ -98,9 +108,6 @@ export async function generateFlowLockInternal(
     );
   }
 
-  const flowValue = (await yamlParseFile(
-    folder! + SEP + "flow.yaml"
-  )) as FlowFile;
 
   if (!justUpdateMetadataLock) {
     const changedScripts = [];
@@ -131,7 +138,7 @@ export async function generateFlowLockInternal(
       workspace,
       flowValue.value,
       remote_path,
-      rawWorkspaceDependencies
+      filteredDeps
     );
 
     const inlineScripts = extractInlineScriptsForFlows(
@@ -152,7 +159,7 @@ export async function generateFlowLockInternal(
   }
 
   hashes = await generateFlowHash(
-    rawWorkspaceDependencies,
+    filteredDeps,
     folder,
     opts.defaultTs
   );
@@ -163,6 +170,25 @@ export async function generateFlowLockInternal(
   log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
 }
 
+/**
+ * Filters raw workspace dependencies for a flow by extracting all inline scripts,
+ * filtering deps for each based on language and annotations, then computing the union.
+ */
+async function filterWorkspaceDependenciesForFlow(
+  flowValue: FlowValue,
+  rawWorkspaceDependencies: Record<string, string>,
+  folder: string
+): Promise<Record<string, string>> {
+  const inlineScripts = extractInlineScriptsForFlows(structuredClone(flowValue.modules), {}, SEP, undefined);
+
+  // Filter out lock files and map to common interface
+  const scripts = inlineScripts
+    .filter(s => !s.is_lock)
+    .map(s => ({ content: s.content, language: s.language as ScriptLanguage }));
+
+  return await filterWorkspaceDependenciesForScripts(scripts, rawWorkspaceDependencies, folder, SEP);
+}
+
 export async function updateFlow(
   workspace: Workspace,
   flow_value: FlowValue,
@@ -191,10 +217,7 @@ export async function updateFlow(
           flow_value,
           path: remotePath,
           use_local_lockfiles: true,
-          raw_workspace_dependencies:
-            Object.keys(rawWorkspaceDependencies).length > 0
-              ? rawWorkspaceDependencies
-              : null,
+          raw_workspace_dependencies: rawWorkspaceDependencies,
         }),
       }
     );

cli/src/commands/sync/sync.ts

@@ -2018,7 +2018,7 @@ export async function pull(
       log.info(`Updating lock metadata for raw app ${change}`);
       await generateAppLocksInternal(
         change,
-        false,
+        true,
         true,
         workspace,
         opts,

cli/src/utils/metadata.ts

@@ -54,19 +54,90 @@ export async function getRawWorkspaceDependencies(): Promise<Record<string, stri
   } catch {
     // dependencies directory doesn't exist
   }
-    return rawWorkspaceDeps;
+  return rawWorkspaceDeps;
 }
 
 export function workspaceDependenciesPathToLanguageAndFilename(path: string): { name: string | undefined, language: ScriptLanguage } | undefined {
-    const relativePath = path.replace("dependencies/", "");
-    for (const { filename, language } of workspaceDependenciesLanguages) {
-      if (relativePath.endsWith(filename)) {
-        return {
-          name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""),
-          language
-        };
+  const relativePath = path.replace("dependencies/", "");
+  for (const { filename, language } of workspaceDependenciesLanguages) {
+    if (relativePath.endsWith(filename)) {
+      return {
+        name: relativePath === filename ? undefined : relativePath.replace("." + filename, ""),
+        language
+      };
+    }
+  }
+}
+
+/**
+ * Filters raw workspace dependencies to only include those that:
+ * 1. Match the given script language
+ * 2. Are referenced by the script's workspace dependency annotation, OR
+ *    are the default dependency (no name) when there's no annotation
+ */
+export function filterWorkspaceDependencies(
+  rawWorkspaceDependencies: Record<string, string>,
+  scriptContent: string,
+  language: ScriptLanguage
+): Record<string, string> {
+  const wda = extractWorkspaceDepsAnnotation(scriptContent, language);
+  const filtered: Record<string, string> = {};
+
+  for (const [depPath, depContent] of Object.entries(rawWorkspaceDependencies)) {
+    const depInfo = workspaceDependenciesPathToLanguageAndFilename(depPath);
+
+    if (depInfo && depInfo.language === language) {
+      if ((wda && wda.external.includes(depInfo.name ?? "default")) || (wda == null && depInfo.name == undefined)) {
+        filtered[depPath] = depContent;
       }
     }
+  }
+
+  return filtered;
+}
+
+export interface InlineScriptInfo {
+  content: string;
+  language: ScriptLanguage;
+}
+
+/**
+ * Filters workspace dependencies for multiple scripts, resolving !inline refs and computing union.
+ * Common helper used by flows and apps.
+ */
+export async function filterWorkspaceDependenciesForScripts(
+  scripts: InlineScriptInfo[],
+  rawWorkspaceDependencies: Record<string, string>,
+  folder: string,
+  sep: string
+): Promise<Record<string, string>> {
+  const filtered: Record<string, string> = {};
+
+  for (const script of scripts) {
+    let content = script.content;
+
+    // Resolve !inline reference to actual content
+    if (content.startsWith("!inline ")) {
+      const filePath = folder + sep + content.replace("!inline ", "");
+      try {
+        content = await Deno.readTextFile(filePath);
+      } catch {
+        continue;
+      }
+    }
+
+    const scriptFiltered = filterWorkspaceDependencies(
+      rawWorkspaceDependencies,
+      content,
+      script.language
+    );
+
+    for (const [depPath, depContent] of Object.entries(scriptFiltered)) {
+      filtered[depPath] = depContent;
+    }
+  }
+
+  return filtered;
 }
 
 // on windows, when using powershell, blue is not readable
@@ -95,14 +166,6 @@ export async function generateScriptMetadataInternal(
 
   const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
 
-  // Filter workspace dependencies to only include those matching the script's language
-  const filteredRawWorkspaceDependencies: Record<string, string> = {};
-  for (const [depPath, depContent] of Object.entries(rawWorkspaceDependencies)) {
-    const depInfo = workspaceDependenciesPathToLanguageAndFilename(depPath);
-    if (depInfo && depInfo.language === language) {
-      filteredRawWorkspaceDependencies[depPath] = depContent;
-    }
-  }
 
   const metadataWithType = await parseMetadataFile(
     remotePath,
@@ -112,7 +175,14 @@ export async function generateScriptMetadataInternal(
   // read script content
   const scriptContent = await Deno.readTextFile(scriptPath);
   const metadataContent = await Deno.readTextFile(metadataWithType.path);
-  
+
+  const filteredRawWorkspaceDependencies = filterWorkspaceDependencies(
+    rawWorkspaceDependencies,
+    scriptContent,
+    language
+  );
+
+
   // Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically
   let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent);
 
@@ -671,12 +741,12 @@ export async function parseMetadataFile(
   scriptPath: string,
   generateMetadataIfMissing:
     | (GlobalOptions & {
-        path: string;
-        workspaceRemote: Workspace;
-        schemaOnly?: boolean;
-        rawWorkspaceDependencies: Record<string, string>;
-        codebases: SyncCodebase[]
-      })
+      path: string;
+      workspaceRemote: Workspace;
+      schemaOnly?: boolean;
+      rawWorkspaceDependencies: Record<string, string>;
+      codebases: SyncCodebase[]
+    })
     | undefined
 ): Promise<{ isJson: boolean; payload: any; path: string }> {
   let metadataFilePath = scriptPath + ".script.json";