From 14ce83cef25225d0fa4aabe0840c733454a7e6e2 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Wed, 21 Jan 2026 13:18:35 +0000 Subject: [PATCH] fix(cli): normalize paths in wmill-lock for cross-platform compatibility (#7645) Paths in wmill-lock.yaml are now normalized to use forward slashes, ensuring the lockfile is portable between Windows and Linux. This also applies to paths used in hash computation for flows and apps. - Add normalizeLockPath() helper function - Update v2LockPath() to normalize path and subpath - Normalize paths in generateFlowHash() and generateAppHash() - Add comprehensive tests for path normalization Co-authored-by: Claude Opus 4.5 --- cli/src/commands/app/app_metadata.ts | 4 +- cli/src/commands/flow/flow_metadata.ts | 5 +- cli/src/utils/metadata.ts | 14 +- cli/test/wmill_lock.test.ts | 338 +++++++++++++++++++++++++ 4 files changed, 357 insertions(+), 4 deletions(-) create mode 100644 cli/test/wmill_lock.test.ts diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index 078ecc1e1a..2a4ded06ae 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -15,6 +15,7 @@ import { updateMetadataGlobalLock, inferSchema, getRawWorkspaceDependencies, + normalizeLockPath, } from "../../utils/metadata.ts"; import { ScriptLanguage, @@ -73,7 +74,8 @@ async function generateAppHash( } if (exts.some((e) => f.path.endsWith(e))) { // Embed lock into hash - const relativePath = f.path.replace(runnablesFolder + SEP, ""); + // Normalize path to ensure OS-independent hashing + const relativePath = normalizeLockPath(f.path.replace(runnablesFolder + SEP, "")); hashes[relativePath] = await generateHash( (await f.getContentText()) + JSON.stringify(rawReqs) ); diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index 441832e55f..b0565830d0 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -15,6 +15,7 @@ import { updateMetadataGlobalLock, LockfileGenerationError, getRawWorkspaceDependencies, + normalizeLockPath, } from "../../utils/metadata.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; @@ -39,7 +40,9 @@ async function generateFlowHash( for await (const f of elems.getChildren()) { if (exts.some((e) => f.path.endsWith(e))) { // Embed workspace dependencies into hash - hashes[f.path] = await generateHash( + // Normalize path to ensure OS-independent hashing + const normalizedPath = normalizeLockPath(f.path); + hashes[normalizedPath] = await generateHash( (await f.getContentText()) + JSON.stringify(rawWorkspaceDependencies) ); } diff --git a/cli/src/utils/metadata.ts b/cli/src/utils/metadata.ts index e52830ba6a..6a598e6d73 100644 --- a/cli/src/utils/metadata.ts +++ b/cli/src/utils/metadata.ts @@ -636,6 +636,15 @@ interface Lock { } const WMILL_LOCKFILE = "wmill-lock.yaml"; + +/** + * Normalizes a path to use Linux separators (forward slashes). + * This ensures wmill-lock.yaml is portable across Windows and Linux. + */ +export function normalizeLockPath(p: string): string { + return p.replace(/\\/g, "/"); +} + export async function readLockfile(): Promise { try { const read = await yamlParseFile(WMILL_LOCKFILE); @@ -654,10 +663,11 @@ export async function readLockfile(): Promise { } function v2LockPath(path: string, subpath?: string) { + const normalizedPath = normalizeLockPath(path); if (subpath) { - return `${path}+${subpath}`; + return `${normalizedPath}+${normalizeLockPath(subpath)}`; } else { - return path; + return normalizedPath; } } export async function checkifMetadataUptodate( diff --git a/cli/test/wmill_lock.test.ts b/cli/test/wmill_lock.test.ts new file mode 100644 index 0000000000..f3a635a36e --- /dev/null +++ b/cli/test/wmill_lock.test.ts @@ -0,0 +1,338 @@ +/** + * wmill-lock Tests + * + * Tests the wmill-lock.yaml path normalization functionality. + * Ensures paths are stored with Linux separators and can be correctly + * looked up on both Windows and Linux systems. + */ + +import { assertEquals } from "https://deno.land/std@0.224.0/assert/mod.ts"; +import * as path from "https://deno.land/std@0.224.0/path/mod.ts"; +import { ensureDir } from "https://deno.land/std@0.224.0/fs/mod.ts"; +import { + normalizeLockPath, + readLockfile, + updateMetadataGlobalLock, + checkifMetadataUptodate, + clearGlobalLock, +} from "../src/utils/metadata.ts"; +import { generateHash } from "../src/utils/utils.ts"; +import { yamlStringify, yamlParseFile } from "../deps.ts"; + +// ============================================================================= +// UNIT TESTS - Path Normalization +// ============================================================================= + +Deno.test("normalizeLockPath: converts Windows backslashes to forward slashes", () => { + assertEquals(normalizeLockPath("f\\test\\script"), "f/test/script"); + assertEquals(normalizeLockPath("f\\deeply\\nested\\path\\script"), "f/deeply/nested/path/script"); +}); + +Deno.test("normalizeLockPath: preserves already-normalized paths", () => { + assertEquals(normalizeLockPath("f/test/script"), "f/test/script"); + assertEquals(normalizeLockPath("f/deeply/nested/path/script"), "f/deeply/nested/path/script"); +}); + +Deno.test("normalizeLockPath: handles paths without separators", () => { + assertEquals(normalizeLockPath("script"), "script"); + assertEquals(normalizeLockPath(""), ""); +}); + +Deno.test("normalizeLockPath: handles mixed separators", () => { + assertEquals(normalizeLockPath("f/test\\nested/script"), "f/test/nested/script"); + assertEquals(normalizeLockPath("f\\test/nested\\script"), "f/test/nested/script"); +}); + +// ============================================================================= +// INTEGRATION TESTS - Lock File Operations with Windows Paths +// ============================================================================= + +async function withTempDir(fn: (tempDir: string) => Promise): Promise { + const tempDir = await Deno.makeTempDir({ prefix: "wmill_lock_test_" }); + const originalCwd = Deno.cwd(); + try { + Deno.chdir(tempDir); + await fn(tempDir); + } finally { + Deno.chdir(originalCwd); + await Deno.remove(tempDir, { recursive: true }); + } +} + +Deno.test("wmill-lock: stores paths with Linux separators even when given Windows paths", async () => { + await withTempDir(async (tempDir) => { + // Simulate a Windows-style path + const windowsPath = "f\\flows\\my-flow.flow"; + const hash = "abc123"; + + await updateMetadataGlobalLock(windowsPath, hash); + + // Read the lockfile directly to check stored format + const lockfile = await yamlParseFile("wmill-lock.yaml") as { version: string; locks: Record }; + + // Path should be stored with forward slashes + assertEquals(lockfile.locks["f/flows/my-flow.flow"], hash); + assertEquals(lockfile.locks["f\\flows\\my-flow.flow"], undefined); + }); +}); + +Deno.test("wmill-lock: checkifMetadataUptodate finds paths regardless of separator style", async () => { + await withTempDir(async (tempDir) => { + const linuxPath = "f/scripts/my-script"; + const windowsPath = "f\\scripts\\my-script"; + const hash = "def456"; + + // Store with Linux-style path + await updateMetadataGlobalLock(linuxPath, hash); + + // Should find with Linux-style lookup + const conf = await readLockfile(); + assertEquals(await checkifMetadataUptodate(linuxPath, hash, conf), true); + + // Should also find with Windows-style lookup (simulating Windows usage) + assertEquals(await checkifMetadataUptodate(windowsPath, hash, conf), true); + + // Should not find with wrong hash + assertEquals(await checkifMetadataUptodate(linuxPath, "wrong", conf), false); + assertEquals(await checkifMetadataUptodate(windowsPath, "wrong", conf), false); + }); +}); + +Deno.test("wmill-lock: updateMetadataGlobalLock with subpath normalizes both path and subpath", async () => { + await withTempDir(async (tempDir) => { + const windowsPath = "f\\flows\\my-flow.flow"; + const windowsSubpath = "inline\\script.ts"; + const hash = "ghi789"; + + await updateMetadataGlobalLock(windowsPath, hash, windowsSubpath); + + // Read the lockfile directly + const lockfile = await yamlParseFile("wmill-lock.yaml") as { version: string; locks: Record }; + + // Both path and subpath should use forward slashes + assertEquals(lockfile.locks["f/flows/my-flow.flow+inline/script.ts"], hash); + }); +}); + +Deno.test("wmill-lock: checkifMetadataUptodate with subpath handles Windows separators", async () => { + await withTempDir(async (tempDir) => { + const linuxPath = "f/apps/my-app.app"; + const linuxSubpath = "scripts/button.ts"; + const windowsPath = "f\\apps\\my-app.app"; + const windowsSubpath = "scripts\\button.ts"; + const hash = "jkl012"; + + // Store with Linux-style paths + await updateMetadataGlobalLock(linuxPath, hash, linuxSubpath); + + const conf = await readLockfile(); + + // Should find with Linux-style lookup + assertEquals(await checkifMetadataUptodate(linuxPath, hash, conf, linuxSubpath), true); + + // Should find with Windows-style lookup + assertEquals(await checkifMetadataUptodate(windowsPath, hash, conf, windowsSubpath), true); + + // Should find with mixed-style lookup + assertEquals(await checkifMetadataUptodate(windowsPath, hash, conf, linuxSubpath), true); + assertEquals(await checkifMetadataUptodate(linuxPath, hash, conf, windowsSubpath), true); + }); +}); + +Deno.test("wmill-lock: clearGlobalLock clears paths regardless of separator style", async () => { + await withTempDir(async (tempDir) => { + const basePath = "f/flows/my-flow.flow"; + const subpath1 = "scripts/a.ts"; + const subpath2 = "scripts/b.ts"; + + // Store multiple entries for the same flow + await updateMetadataGlobalLock(basePath, "hash1", subpath1); + await updateMetadataGlobalLock(basePath, "hash2", subpath2); + await updateMetadataGlobalLock(basePath, "topHash", "__flow_hash"); + + // Verify they exist + let conf = await readLockfile(); + assertEquals(await checkifMetadataUptodate(basePath, "hash1", conf, subpath1), true); + assertEquals(await checkifMetadataUptodate(basePath, "hash2", conf, subpath2), true); + + // Clear using Windows-style path + await clearGlobalLock("f\\flows\\my-flow.flow"); + + // All entries should be cleared + conf = await readLockfile(); + assertEquals(await checkifMetadataUptodate(basePath, "hash1", conf, subpath1), false); + assertEquals(await checkifMetadataUptodate(basePath, "hash2", conf, subpath2), false); + assertEquals(await checkifMetadataUptodate(basePath, "topHash", conf, "__flow_hash"), false); + }); +}); + +Deno.test("wmill-lock: lock file created on Linux can be used on Windows (simulated)", async () => { + await withTempDir(async (tempDir) => { + // Simulate a lock file created on Linux + const linuxLockContent = { + version: "v2" as const, + locks: { + "f/scripts/utility": "hash1", + "f/flows/main.flow+scripts/step1.ts": "hash2", + "f/apps/dashboard.app+components/chart.ts": "hash3", + }, + }; + + await Deno.writeTextFile( + "wmill-lock.yaml", + yamlStringify(linuxLockContent as Record) + ); + + const conf = await readLockfile(); + + // Simulate Windows lookups (using backslashes) + assertEquals(await checkifMetadataUptodate("f\\scripts\\utility", "hash1", conf), true); + assertEquals(await checkifMetadataUptodate("f\\flows\\main.flow", "hash2", conf, "scripts\\step1.ts"), true); + assertEquals(await checkifMetadataUptodate("f\\apps\\dashboard.app", "hash3", conf, "components\\chart.ts"), true); + }); +}); + +Deno.test("wmill-lock: multiple updates with different separator styles result in single entry", async () => { + await withTempDir(async (tempDir) => { + const linuxPath = "f/scripts/shared"; + const windowsPath = "f\\scripts\\shared"; + + // Update with Linux-style path + await updateMetadataGlobalLock(linuxPath, "hash1"); + + // Update same path with Windows-style path (should overwrite, not create new entry) + await updateMetadataGlobalLock(windowsPath, "hash2"); + + const lockfile = await yamlParseFile("wmill-lock.yaml") as { version: string; locks: Record }; + + // Should only have one entry with the latest hash + const lockKeys = Object.keys(lockfile.locks); + assertEquals(lockKeys.length, 1); + assertEquals(lockKeys[0], "f/scripts/shared"); + assertEquals(lockfile.locks["f/scripts/shared"], "hash2"); + }); +}); + +// ============================================================================= +// HASH COMPUTATION TESTS - OS-Independent Hash Generation +// ============================================================================= + +Deno.test("hash computation: normalized paths produce same hash on Windows and Linux", async () => { + // Simulate how generateFlowHash/generateAppHash compute hashes + // by using paths as keys in an object that gets stringified + + const fileContents = { + "script1.ts": "export function main() { return 1; }", + "nested/script2.ts": "export function main() { return 2; }", + }; + + // Simulate Windows paths + const windowsHashes: Record = {}; + for (const [relativePath, content] of Object.entries(fileContents)) { + const windowsPath = relativePath.replace(/\//g, "\\"); + // Normalize before using as key (this is what the fix does) + const normalizedPath = normalizeLockPath(windowsPath); + windowsHashes[normalizedPath] = await generateHash(content); + } + const windowsTopHash = await generateHash(JSON.stringify(windowsHashes)); + + // Simulate Linux paths + const linuxHashes: Record = {}; + for (const [relativePath, content] of Object.entries(fileContents)) { + // Linux paths are already normalized + const normalizedPath = normalizeLockPath(relativePath); + linuxHashes[normalizedPath] = await generateHash(content); + } + const linuxTopHash = await generateHash(JSON.stringify(linuxHashes)); + + // Both should produce the same top hash + assertEquals(windowsTopHash, linuxTopHash); + + // And the individual hashes should have the same keys + assertEquals(Object.keys(windowsHashes).sort(), Object.keys(linuxHashes).sort()); +}); + +Deno.test("hash computation: without normalization, Windows and Linux would produce different hashes", async () => { + // This test demonstrates the problem that normalization fixes + const fileContents = { + "script1.ts": "export function main() { return 1; }", + "nested/script2.ts": "export function main() { return 2; }", + }; + + // Simulate Windows paths WITHOUT normalization + const windowsHashesNoNormalize: Record = {}; + for (const [relativePath, content] of Object.entries(fileContents)) { + const windowsPath = relativePath.replace(/\//g, "\\"); + // NOT normalizing - simulating the old behavior + windowsHashesNoNormalize[windowsPath] = await generateHash(content); + } + const windowsTopHashNoNormalize = await generateHash(JSON.stringify(windowsHashesNoNormalize)); + + // Simulate Linux paths WITHOUT normalization + const linuxHashesNoNormalize: Record = {}; + for (const [relativePath, content] of Object.entries(fileContents)) { + // NOT normalizing - simulating the old behavior + linuxHashesNoNormalize[relativePath] = await generateHash(content); + } + const linuxTopHashNoNormalize = await generateHash(JSON.stringify(linuxHashesNoNormalize)); + + // Without normalization, the hashes WOULD be different (this is the bug we fixed) + // The keys are different: "nested\\script2.ts" vs "nested/script2.ts" + const windowsKeys = Object.keys(windowsHashesNoNormalize).sort(); + const linuxKeys = Object.keys(linuxHashesNoNormalize).sort(); + + // Keys should be different without normalization + assertEquals(windowsKeys.includes("nested\\script2.ts"), true); + assertEquals(linuxKeys.includes("nested/script2.ts"), true); + assertEquals(windowsKeys.includes("nested/script2.ts"), false); + assertEquals(linuxKeys.includes("nested\\script2.ts"), false); +}); + +Deno.test("hash computation: deeply nested paths are normalized correctly", async () => { + const deepWindowsPath = "f\\flows\\my-flow.flow\\inline\\scripts\\deeply\\nested\\handler.ts"; + const deepLinuxPath = "f/flows/my-flow.flow/inline/scripts/deeply/nested/handler.ts"; + + const content = "export function main() { return 'deeply nested'; }"; + + // Hash with Windows path (normalized) + const windowsHashes: Record = {}; + windowsHashes[normalizeLockPath(deepWindowsPath)] = await generateHash(content); + const windowsTopHash = await generateHash(JSON.stringify(windowsHashes)); + + // Hash with Linux path (normalized) + const linuxHashes: Record = {}; + linuxHashes[normalizeLockPath(deepLinuxPath)] = await generateHash(content); + const linuxTopHash = await generateHash(JSON.stringify(linuxHashes)); + + assertEquals(windowsTopHash, linuxTopHash); + assertEquals(Object.keys(windowsHashes)[0], Object.keys(linuxHashes)[0]); + assertEquals(Object.keys(windowsHashes)[0], deepLinuxPath); +}); + +Deno.test("hash computation: changedScripts comparison works with inline module paths", () => { + // This test simulates the comparison done in replaceInlineScripts + // where changedScripts (from hashes keys) is compared with paths from flow module content + + // Simulate changedScripts populated from hashes (normalized from Windows paths) + const changedScripts = [ + normalizeLockPath("step1.ts"), + normalizeLockPath("nested\\step2.ts"), // Windows-style from FSFSElement + normalizeLockPath("deeply\\nested\\step3.ts"), + ]; + + // Simulate paths extracted from flow module content (!inline paths are always forward slashes) + const inlineModulePaths = [ + "step1.ts", + "nested/step2.ts", // Forward slashes as stored in flow.yaml + "deeply/nested/step3.ts", + ]; + + // All inline module paths should be found in changedScripts + for (const inlinePath of inlineModulePaths) { + assertEquals( + changedScripts.includes(inlinePath), + true, + `Expected changedScripts to include "${inlinePath}"` + ); + } +});