diff --git a/cli/deno.lock b/cli/deno.lock index 9fcd2b7a64..03eca92bfd 100644 --- a/cli/deno.lock +++ b/cli/deno.lock @@ -1,7 +1,7 @@ { "version": "5", "specifiers": { - "jsr:@david/code-block-writer@^13.0.2": "13.0.2", + "jsr:@david/code-block-writer@^13.0.2": "13.0.3", "jsr:@david/code-block-writer@^13.0.3": "13.0.3", "jsr:@deno/cache-dir@~0.10.3": "0.10.3", "jsr:@deno/dnt@0.41.3": "0.41.3", @@ -12,10 +12,13 @@ "jsr:@std/assert@0.226": "0.226.0", "jsr:@std/assert@1.0.0-rc.2": "1.0.0-rc.2", "jsr:@std/bytes@0.223": "0.223.0", + "jsr:@std/bytes@^1.0.2": "1.0.6", "jsr:@std/bytes@^1.0.5": "1.0.6", + "jsr:@std/bytes@^1.0.6": "1.0.6", "jsr:@std/cli@1.0.0-rc.2": "1.0.0-rc.2", "jsr:@std/encoding@1.0.0-rc.2": "1.0.0-rc.2", "jsr:@std/encoding@1.0.4": "1.0.4", + "jsr:@std/encoding@^1.0.10": "1.0.10", "jsr:@std/fmt@0.223": "0.223.0", "jsr:@std/fmt@1": "1.0.8", "jsr:@std/fmt@^1.0.5": "1.0.8", @@ -30,8 +33,11 @@ "jsr:@std/io@*": "0.225.2", "jsr:@std/io@0.223": "0.223.0", "jsr:@std/io@~0.224.2": "0.224.9", + "jsr:@std/io@~0.224.9": "0.224.9", "jsr:@std/io@~0.225.2": "0.225.2", "jsr:@std/log@*": "0.224.14", + "jsr:@std/log@~0.224.14": "0.224.14", + "jsr:@std/net@^1.0.6": "1.0.6", "jsr:@std/path@*": "1.1.4", "jsr:@std/path@0.223": "0.223.0", "jsr:@std/path@1": "1.1.4", @@ -40,6 +46,7 @@ "jsr:@std/path@^1.1.3": "1.1.4", "jsr:@std/path@^1.1.4": "1.1.4", "jsr:@std/path@~0.225.2": "0.225.2", + "jsr:@std/streams@^1.0.16": "1.0.17", "jsr:@std/text@1.0.0-rc.1": "1.0.0-rc.1", "jsr:@std/yaml@*": "1.0.10", "jsr:@std/yaml@^1.0.10": "1.0.10", @@ -48,12 +55,16 @@ "jsr:@ts-morph/common@0.24": "0.24.0", "jsr:@ts-morph/common@0.27": "0.27.0", "jsr:@windmill-labs/cliffy-ansi@1.0.0-rc.5": "1.0.0-rc.5", + "jsr:@windmill-labs/cliffy-ansi@^1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/cliffy-command@1.0.0-rc.5": "1.0.0-rc.5", + "jsr:@windmill-labs/cliffy-command@^1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/cliffy-flags@1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/cliffy-internal@1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/cliffy-keycode@1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/cliffy-prompt@1.0.0-rc.6": "1.0.0-rc.6", + "jsr:@windmill-labs/cliffy-prompt@^1.0.0-rc.6": "1.0.0-rc.6", "jsr:@windmill-labs/cliffy-table@1.0.0-rc.5": "1.0.0-rc.5", + "jsr:@windmill-labs/cliffy-table@^1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/shared-utils@1.0.10": "1.0.10", "jsr:@windmill-labs/shared-utils@1.0.11": "1.0.11", "jsr:@windmill-labs/shared-utils@1.0.12": "1.0.12", @@ -157,6 +168,9 @@ "@std/encoding@1.0.4": { "integrity": "2266cd516b32369e3dc5695717c96bf88343a1f761d6e6187a02a2bbe2af86ae" }, + "@std/encoding@1.0.10": { + "integrity": "8783c6384a2d13abd5e9e87a7ae0520a30e9f56aeeaa3bdf910a3eaaf5c811a1" + }, "@std/fmt@0.223.0": { "integrity": "6deb37794127dfc7d7bded2586b9fc6f5d50e62a8134846608baf71ffc1a5208" }, @@ -200,7 +214,10 @@ ] }, "@std/io@0.224.9": { - "integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3" + "integrity": "4414664b6926f665102e73c969cfda06d2c4c59bd5d0c603fd4f1b1c840d6ee3", + "dependencies": [ + "jsr:@std/bytes@^1.0.2" + ] }, "@std/io@0.225.2": { "integrity": "3c740cd4ee4c082e6cfc86458f47e2ab7cb353dc6234d5e9b1f91a2de5f4d6c7", @@ -216,6 +233,9 @@ "jsr:@std/io@~0.225.2" ] }, + "@std/net@1.0.6": { + "integrity": "110735f93e95bb9feb95790a8b1d1bf69ec0dc74f3f97a00a76ea5efea25500c" + }, "@std/path@0.223.0": { "integrity": "593963402d7e6597f5a6e620931661053572c982fc014000459edc1f93cc3989", "dependencies": [ @@ -246,6 +266,12 @@ "jsr:@std/internal" ] }, + "@std/streams@1.0.17": { + "integrity": "7859f3d9deed83cf4b41f19223d4a67661b3d3819e9fc117698f493bf5992140", + "dependencies": [ + "jsr:@std/bytes@^1.0.6" + ] + }, "@std/text@1.0.0-rc.1": { "integrity": "34c722203e87ee12792c8d4a0cd2ee0e001341cbce75b860fc21be19d62232b0" }, @@ -294,7 +320,7 @@ "jsr:@std/text", "jsr:@windmill-labs/cliffy-flags", "jsr:@windmill-labs/cliffy-internal", - "jsr:@windmill-labs/cliffy-table" + "jsr:@windmill-labs/cliffy-table@1.0.0-rc.5" ] }, "@windmill-labs/cliffy-flags@1.0.0-rc.5": { @@ -317,7 +343,7 @@ "jsr:@std/io@~0.224.2", "jsr:@std/path@1.0.0-rc.2", "jsr:@std/text", - "jsr:@windmill-labs/cliffy-ansi", + "jsr:@windmill-labs/cliffy-ansi@1.0.0-rc.5", "jsr:@windmill-labs/cliffy-internal", "jsr:@windmill-labs/cliffy-keycode" ] diff --git a/cli/src/commands/lint/lint.ts b/cli/src/commands/lint/lint.ts index 58d456bed0..b281587675 100644 --- a/cli/src/commands/lint/lint.ts +++ b/cli/src/commands/lint/lint.ts @@ -1,4 +1,11 @@ -import { colors, Command, log, path, SEP } from "../../../deps.ts"; +import { + colors, + Command, + log, + path, + SEP, + yamlParseFile, +} from "../../../deps.ts"; import { GlobalOptions } from "../../types.ts"; import { mergeConfigWithConfigFile } from "../../core/conf.ts"; import { @@ -11,10 +18,23 @@ import { type ValidationTarget, WindmillYamlValidator, } from "npm:windmill-yaml-validator@1.1.1"; +import { + inferContentTypeFromFilePath, + languageNeedsLock, + ScriptLanguage, +} from "../../utils/script_common.ts"; +import { + isFlowInlineScriptPath, + isAppInlineScriptPath, + isRawAppPath, + getFolderSuffix, +} from "../../utils/resource_folders.ts"; +import { exts } from "../script/script.ts"; interface LintOptions extends GlobalOptions { json?: boolean; failOnWarn?: boolean; + locksRequired?: boolean; } interface FileIssue { @@ -101,6 +121,487 @@ function formatYamlDiagnostics(parsed: { diagnostics?: Array<{ message?: string return diagnostics.map((d) => d?.message || "Invalid YAML document"); } +/** + * Check if a lock value represents an actually resolved lock. + * Returns true if the lock is present and valid, false if missing. + * For `!inline` references, checks that the referenced file exists and is non-empty. + */ +async function isLockResolved( + lockValue: string | string[] | undefined, + baseDir: string, +): Promise { + if (lockValue === undefined) return false; + + // Array lock (v2 format) - if non-empty, locks are present + if (Array.isArray(lockValue)) { + const joined = lockValue.join("\n"); + if (joined === "") return false; + if (joined.startsWith("!inline ")) { + return await checkInlineFile(joined.substring("!inline ".length), baseDir); + } + return true; + } + + if (lockValue === "") return false; + + // Inline file reference + if (lockValue.startsWith("!inline ")) { + return await checkInlineFile(lockValue.substring("!inline ".length), baseDir); + } + + // Embedded lock content + return true; +} + +async function checkInlineFile( + relativePath: string, + baseDir: string, +): Promise { + const fullPath = path.join(baseDir, relativePath.trim()); + try { + const stat = await Deno.stat(fullPath); + return stat.size > 0; + } catch { + return false; + } +} + +/** + * Recursively find rawscript modules in a flow's module tree. + */ +function findRawScriptsInModules( + modules: any[], +): { language: string; lock: any; id: string }[] { + const results: { language: string; lock: any; id: string }[] = []; + if (!modules || !Array.isArray(modules)) return results; + + for (const m of modules) { + if (!m?.value?.type) continue; + + if (m.value.type === "rawscript") { + results.push({ + language: m.value.language, + lock: m.value.lock, + id: m.id ?? "unknown", + }); + } else if ( + m.value.type === "forloopflow" || + m.value.type === "whileloopflow" + ) { + results.push(...findRawScriptsInModules(m.value.modules)); + } else if (m.value.type === "branchall") { + for (const b of m.value.branches ?? []) { + results.push(...findRawScriptsInModules(b.modules)); + } + } else if (m.value.type === "branchone") { + for (const b of m.value.branches ?? []) { + results.push(...findRawScriptsInModules(b.modules)); + } + if (m.value.default) { + results.push(...findRawScriptsInModules(m.value.default)); + } + } else if (m.value.type === "aiagent") { + for (const tool of m.value.tools ?? []) { + const toolValue = tool.value; + if ( + toolValue?.tool_type === "flowmodule" && + toolValue?.type === "rawscript" + ) { + results.push({ + language: toolValue.language, + lock: toolValue.lock, + id: tool.id ?? "unknown", + }); + } + } + } + } + + return results; +} + +/** + * Recursively find inlineScript objects in a normal app's value structure. + * Follows the same traversal as traverseAndProcessInlineScripts in app_metadata.ts. + */ +function findInlineScriptsInApp( + obj: any, + currentPath: string[] = [], +): { language: string; lock: any; path: string }[] { + const results: { language: string; lock: any; path: string }[] = []; + if (!obj || typeof obj !== "object") return results; + + if (Array.isArray(obj)) { + for (let i = 0; i < obj.length; i++) { + results.push( + ...findInlineScriptsInApp(obj[i], [...currentPath, `[${i}]`]), + ); + } + return results; + } + + for (const [key, value] of Object.entries(obj)) { + if (key === "inlineScript" && typeof value === "object" && value !== null) { + const script = value as Record; + if (script.language) { + results.push({ + language: script.language, + lock: script.lock, + path: [...currentPath, key].join("."), + }); + } + } else { + results.push( + ...findInlineScriptsInApp(value, [...currentPath, key]), + ); + } + } + + return results; +} + +/** + * Check raw app backend runnables for missing locks. + * Reads YAML config files and code files from the backend/ folder. + */ +async function checkRawAppRunnables( + backendDir: string, + rawAppYamlPath: string, + defaultTs: "bun" | "deno" | undefined, +): Promise { + const issues: FileIssue[] = []; + + const allFiles: string[] = []; + for await (const entry of Deno.readDir(backendDir)) { + if (entry.isFile) { + allFiles.push(entry.name); + } + } + + // Track processed IDs to avoid duplicates + const processedIds = new Set(); + + // Process YAML files (explicit config) + for (const fileName of allFiles) { + if (!fileName.endsWith(".yaml")) continue; + + const runnableId = fileName.replace(".yaml", ""); + processedIds.add(runnableId); + + const filePath = path.join(backendDir, fileName); + let runnable: Record; + try { + runnable = (await yamlParseFile(filePath)) as Record; + } catch { + continue; + } + + // Only inline runnables need lock checking + if (runnable?.type !== "inline") continue; + + // Find the content file to determine language + let language: string | null = null; + for (const codeFile of allFiles) { + if ( + codeFile.endsWith(".yaml") || codeFile.endsWith(".lock") || + !codeFile.startsWith(runnableId + ".") + ) continue; + language = inferContentTypeFromFilePath(codeFile, defaultTs); + break; + } + + if (!language || !languageNeedsLock(language)) continue; + + // Check for lock file + const lockFile = path.join(backendDir, `${runnableId}.lock`); + let hasLock = false; + try { + const stat = await Deno.stat(lockFile); + hasLock = stat.size > 0; + } catch { + // No lock file + } + + // Also check if the runnable YAML has inlineScript.lock + if (!hasLock && runnable.inlineScript?.lock) { + hasLock = await isLockResolved(runnable.inlineScript.lock, backendDir); + } + + if (!hasLock) { + issues.push({ + path: rawAppYamlPath, + target: "raw_app_inline_script", + errors: [ + `Missing lock for ${language} runnable '${runnableId}'. Run 'wmill app generate-locks' to generate locks.`, + ], + }); + } + } + + // Auto-detect code files without YAML config + for (const fileName of allFiles) { + if (fileName.endsWith(".yaml") || fileName.endsWith(".lock")) continue; + + // Extract runnableId from code file + let runnableId: string | null = null; + try { + const lang = inferContentTypeFromFilePath(fileName, defaultTs); + if (lang) { + // The runnableId is the filename without the extension portion + // We need to find which extension matches + for (const ext of exts) { + if (fileName.endsWith(ext)) { + runnableId = fileName.slice(0, -ext.length); + break; + } + } + } + } catch { + continue; + } + + if (!runnableId || processedIds.has(runnableId)) continue; + processedIds.add(runnableId); + + let language: string; + try { + language = inferContentTypeFromFilePath(fileName, defaultTs); + } catch { + continue; + } + + if (!languageNeedsLock(language)) continue; + + const lockFile = path.join(backendDir, `${runnableId}.lock`); + let hasLock = false; + try { + const stat = await Deno.stat(lockFile); + hasLock = stat.size > 0; + } catch { + // No lock file + } + + if (!hasLock) { + issues.push({ + path: rawAppYamlPath, + target: "raw_app_inline_script", + errors: [ + `Missing lock for ${language} runnable '${runnableId}'. Run 'wmill app generate-locks' to generate locks.`, + ], + }); + } + } + + return issues; +} + +/** + * Check for missing lock files across scripts, flow inline scripts, + * app inline scripts, and raw app backend scripts. + * Returns a list of issues for scripts/inline scripts that should have locks but don't. + */ +export async function checkMissingLocks( + opts: GlobalOptions & { defaultTs?: "bun" | "deno" }, + directory?: string, +): Promise { + const initialCwd = Deno.cwd(); + const targetDirectory = directory + ? path.resolve(initialCwd, directory) + : Deno.cwd(); + + const { ...syncOpts } = opts; + const mergedOpts = await mergeConfigWithConfigFile(syncOpts); + + const ignore = await ignoreF(mergedOpts); + const root = await FSFSElement(targetDirectory, [], false); + + const issues: FileIssue[] = []; + const defaultTs = mergedOpts.defaultTs; + const flowSuffix = getFolderSuffix("flow"); + const appSuffix = getFolderSuffix("app"); + const rawAppSuffix = getFolderSuffix("raw_app"); + + // Collect all file paths and categorize them + const scriptYamls: string[] = []; + const flowYamls: { normalizedPath: string; fullPath: string }[] = []; + const appYamls: { normalizedPath: string; fullPath: string }[] = []; + const rawAppYamls: { normalizedPath: string; fullPath: string }[] = []; + + for await (const entry of readDirRecursiveWithIgnore(ignore, root)) { + if (entry.isDirectory || entry.ignored) continue; + + const normalizedPath = normalizePath(entry.path); + + // Standalone script metadata files (not inside flow/app folders) + if ( + normalizedPath.endsWith(".script.yaml") && + !isFlowInlineScriptPath(normalizedPath) && + !isAppInlineScriptPath(normalizedPath) + ) { + scriptYamls.push(normalizedPath); + } + + // Flow definition files + if ( + normalizedPath.endsWith("/flow.yaml") && + normalizedPath.includes(flowSuffix + "/") + ) { + flowYamls.push({ + normalizedPath, + fullPath: path.join(targetDirectory, entry.path), + }); + } + + // Normal app definition files + if ( + normalizedPath.endsWith("/app.yaml") && + normalizedPath.includes(appSuffix + "/") + ) { + appYamls.push({ + normalizedPath, + fullPath: path.join(targetDirectory, entry.path), + }); + } + + // Raw app definition files + if ( + normalizedPath.endsWith("/raw_app.yaml") && + normalizedPath.includes(rawAppSuffix + "/") + ) { + rawAppYamls.push({ + normalizedPath, + fullPath: path.join(targetDirectory, entry.path), + }); + } + } + + // Check standalone scripts + for (const yamlPath of scriptYamls) { + const basePath = yamlPath.replace(/\.script\.yaml$/, ""); + + // Find the content file to determine language + let language: ScriptLanguage | null = null; + for (const ext of exts) { + try { + await Deno.stat(path.join(targetDirectory, basePath + ext)); + language = inferContentTypeFromFilePath(basePath + ext, defaultTs); + break; + } catch { + // Content file with this extension doesn't exist, try next + } + } + + if (language && languageNeedsLock(language)) { + // Read the metadata to check the lock field + try { + const metadata = (await yamlParseFile( + path.join(targetDirectory, yamlPath), + )) as { lock?: string | string[] }; + + const lockResolved = await isLockResolved( + metadata?.lock, + targetDirectory, + ); + if (!lockResolved) { + issues.push({ + path: yamlPath, + target: "script", + errors: [ + `Missing lock for ${language} script. Run 'wmill script generate-metadata' to generate locks.`, + ], + }); + } + } catch (e) { + log.debug(`Failed to parse ${yamlPath}: ${e}`); + } + } + } + + // Check flow inline scripts + for (const { normalizedPath: flowYamlPath, fullPath } of flowYamls) { + const flowDir = path.dirname(fullPath); + + try { + const flowFile = (await yamlParseFile(fullPath)) as { + value?: { modules?: any[] }; + }; + if (!flowFile?.value?.modules) continue; + + const rawScripts = findRawScriptsInModules(flowFile.value.modules); + + for (const script of rawScripts) { + if (!languageNeedsLock(script.language as ScriptLanguage)) continue; + + const lockResolved = await isLockResolved(script.lock, flowDir); + if (!lockResolved) { + issues.push({ + path: flowYamlPath, + target: "flow_inline_script", + errors: [ + `Missing lock for ${script.language} inline script '${script.id}'. Run 'wmill flow generate-locks' to generate locks.`, + ], + }); + } + } + } catch (e) { + log.debug(`Failed to parse flow ${flowYamlPath}: ${e}`); + } + } + + // Check normal app inline scripts + for (const { normalizedPath: appYamlPath, fullPath } of appYamls) { + const appDir = path.dirname(fullPath); + + try { + const appFile = (await yamlParseFile(fullPath)) as { value?: any }; + if (!appFile?.value) continue; + + const inlineScripts = findInlineScriptsInApp(appFile.value); + for (const script of inlineScripts) { + if (!languageNeedsLock(script.language)) continue; + + const lockResolved = await isLockResolved(script.lock, appDir); + if (!lockResolved) { + issues.push({ + path: appYamlPath, + target: "app_inline_script", + errors: [ + `Missing lock for ${script.language} inline script at '${script.path}'. Run 'wmill app generate-locks' to generate locks.`, + ], + }); + } + } + } catch (e) { + log.debug(`Failed to parse app ${appYamlPath}: ${e}`); + } + } + + // Check raw app backend scripts + for (const { normalizedPath: rawAppYamlPath, fullPath } of rawAppYamls) { + const rawAppDir = path.dirname(fullPath); + const backendDir = path.join(rawAppDir, "backend"); + + try { + await Deno.stat(backendDir); + } catch { + continue; // No backend folder + } + + try { + const runnableIssues = await checkRawAppRunnables( + backendDir, + rawAppYamlPath, + defaultTs, + ); + issues.push(...runnableIssues); + } catch (e) { + log.debug(`Failed to check raw app runnables ${rawAppYamlPath}: ${e}`); + } + } + + return issues; +} + export async function runLint( opts: LintOptions, directory?: string, @@ -178,6 +679,12 @@ export async function runLint( } } + // Check for missing locks if --locks-required is set + if (opts.locksRequired) { + const lockIssues = await checkMissingLocks(opts, explicitTargetDirectory); + issues.push(...lockIssues); + } + const invalidFiles = issues.length; const shouldFail = invalidFiles > 0 || (!!opts.failOnWarn && warnings.length > 0); @@ -268,6 +775,10 @@ const command = new Command() .arguments("[directory:string]") .option("--json", "Output results in JSON format") .option("--fail-on-warn", "Exit with code 1 when warnings are emitted") + .option( + "--locks-required", + "Fail if scripts or flow inline scripts that need locks have no locks", + ) .action(lint as any); export default command; diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index b1b6142ce5..41af0159d2 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -25,7 +25,7 @@ import { extractNativeTriggerInfo, } from "../../types.ts"; import { downloadZip } from "./pull.ts"; -import { runLint, printReport } from "../lint/lint.ts"; +import { runLint, printReport, checkMissingLocks } from "../lint/lint.ts"; import { exts, @@ -2221,6 +2221,25 @@ export async function push( } } + if (opts.locksRequired) { + log.info("Checking for missing locks..."); + const lockIssues = await checkMissingLocks(opts); + if (lockIssues.length > 0) { + for (const issue of lockIssues) { + for (const error of issue.errors) { + log.error(colors.red(` ${issue.path}: ${error}`)); + } + } + log.error( + colors.red( + `\nPush aborted: ${lockIssues.length} script(s) missing locks.`, + ), + ); + Deno.exit(1); + } + log.info(colors.green("All scripts have valid locks.")); + } + const codebases = await listSyncCodebases(opts); if (opts.raw) { log.info("--raw is now the default, you can remove it as a flag"); @@ -3090,6 +3109,10 @@ const command = new Command() "Override the current git branch (works even outside a git repository)", ) .option("--lint", "Run lint validation before pushing") + .option( + "--locks-required", + "Fail if scripts or flow inline scripts that need locks have no locks", + ) // deno-lint-ignore no-explicit-any .action(push as any); diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index dd38adbd0d..ce6c4f5bf1 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -97,6 +97,7 @@ export interface SyncOptions { }; promotion?: string; lint?: boolean; + locksRequired?: boolean; } export interface Codebase { diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index a5aba792c8..558a83bda6 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -39,6 +39,19 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [ { language: "go", filename: "go.mod" }, ] as const; +/** + * Returns true if a script in the given language requires a lock file. + * Matches the condition in updateScriptLock (metadata.ts). + */ +export function languageNeedsLock(language: ScriptLanguage | string): boolean { + return ( + workspaceDependenciesLanguages.some((l) => l.language === language) || + language === "deno" || + language === "rust" || + language === "ansible" + ); +} + export function inferContentTypeFromFilePath( contentPath: string, defaultTs: "bun" | "deno" | undefined diff --git a/cli/test/locks_required.test.ts b/cli/test/locks_required.test.ts new file mode 100644 index 0000000000..150c24593c --- /dev/null +++ b/cli/test/locks_required.test.ts @@ -0,0 +1,620 @@ +import { + assert, + assertEquals, +} from "https://deno.land/std@0.224.0/assert/mod.ts"; +import { + checkMissingLocks, + runLint, +} from "../src/commands/lint/lint.ts"; + +async function withTempDir( + fn: (tempDir: string) => Promise, +): Promise { + const tempDir = await Deno.makeTempDir({ prefix: "wmill_locks_test_" }); + const originalCwd = Deno.cwd(); + try { + Deno.chdir(tempDir); + await fn(tempDir); + } finally { + Deno.chdir(originalCwd); + await Deno.remove(tempDir, { recursive: true }); + } +} + +// --- checkMissingLocks unit tests --- + +Deno.test("locks-required: passes for python script with non-empty lock file", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.py`, + `import pandas\ndef main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: "!inline f/folder/my_script.script.lock"\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.lock`, + `pandas==2.0.0\nnumpy==1.24.0\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: fails for python script with empty lock file", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: "!inline f/folder/my_script.script.lock"\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.lock`, + ``, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "script"); + assert(issues[0].errors[0].includes("Missing lock")); + assert(issues[0].errors[0].includes("python3")); + }); +}); + +Deno.test("locks-required: fails for python script with missing lock file", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: "!inline f/folder/nonexistent.script.lock"\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "script"); + }); +}); + +Deno.test("locks-required: fails for python script with lock field empty string", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "script"); + }); +}); + +Deno.test("locks-required: skips bash scripts (no locks needed)", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.sh`, + `#!/bin/bash\necho hello`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: skips SQL scripts (no locks needed)", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_query.pg.sql`, + `SELECT 1;`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_query.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: checks bun typescript scripts", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.bun.ts`, + `export async function main() { return "hello"; }`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assert(issues[0].errors[0].includes("bun")); + }); +}); + +Deno.test("locks-required: checks deno typescript scripts", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.deno.ts`, + `export async function main() { return "hello"; }`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assert(issues[0].errors[0].includes("deno")); + }); +}); + +// --- Flow inline script tests --- + +Deno.test("locks-required: fails for flow with unlocked inline python script", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_flow.flow`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/inline_script_0.inline_script.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/flow.yaml`, + `summary: My flow +value: + modules: + - id: a + value: + type: rawscript + language: python3 + content: "!inline inline_script_0.inline_script.py" + lock: "" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "flow_inline_script"); + assert(issues[0].errors[0].includes("python3")); + assert(issues[0].errors[0].includes("'a'")); + }); +}); + +Deno.test("locks-required: passes for flow with locked inline python script", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_flow.flow`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/inline_script_0.inline_script.py`, + `import pandas\ndef main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/inline_script_0.inline_script.lock`, + `pandas==2.0.0\n`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/flow.yaml`, + `summary: My flow +value: + modules: + - id: a + value: + type: rawscript + language: python3 + content: "!inline inline_script_0.inline_script.py" + lock: "!inline inline_script_0.inline_script.lock" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: skips flow inline bash scripts", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_flow.flow`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/flow.yaml`, + `summary: My flow +value: + modules: + - id: a + value: + type: rawscript + language: bash + content: "echo hello" + lock: "" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: checks nested flow modules (forloopflow)", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_flow.flow`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/flow.yaml`, + `summary: My flow +value: + modules: + - id: loop + value: + type: forloopflow + modules: + - id: inner + value: + type: rawscript + language: python3 + content: "def main(): pass" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assert(issues[0].errors[0].includes("'inner'")); + }); +}); + +Deno.test("locks-required: checks nested flow modules (branchone)", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_flow.flow`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_flow.flow/flow.yaml`, + `summary: My flow +value: + modules: + - id: branch + value: + type: branchone + branches: + - modules: + - id: branch_script + value: + type: rawscript + language: bun + content: "export async function main() {}" + default: + - id: default_script + value: + type: rawscript + language: python3 + content: "def main(): pass" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 2); + const ids = issues.map((i) => i.errors[0]); + assert(ids.some((e) => e.includes("'branch_script'"))); + assert(ids.some((e) => e.includes("'default_script'"))); + }); +}); + +// --- Integration with runLint --- + +Deno.test("locks-required: runLint includes lock issues when flag is set", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const report = await runLint({ locksRequired: true } as any, tempDir); + assertEquals(report.success, false); + assertEquals(report.exitCode, 1); + assert(report.issues.some((i) => i.target === "script")); + }); +}); + +Deno.test("locks-required: runLint skips lock check when flag is not set", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/my_script.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const report = await runLint({} as any, tempDir); + assertEquals(report.success, true); + assertEquals(report.exitCode, 0); + assertEquals(report.issues.length, 0); + }); +}); + +// --- Multiple scripts --- + +Deno.test("locks-required: reports multiple missing locks", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/folder`, { recursive: true }); + + // Python script without lock + await Deno.writeTextFile( + `${tempDir}/f/folder/script1.py`, + `def main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/script1.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + // Go script without lock + await Deno.writeTextFile( + `${tempDir}/f/folder/script2.go`, + `package main\nfunc main() {}`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/script2.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + // Bash script (should pass - no lock needed) + await Deno.writeTextFile( + `${tempDir}/f/folder/script3.sh`, + `#!/bin/bash\necho ok`, + ); + await Deno.writeTextFile( + `${tempDir}/f/folder/script3.script.yaml`, + `summary: ""\ndescription: ""\nlock: ""\nkind: script\nschema:\n $schema: "https://json-schema.org/draft/2020-12/schema"\n type: object\n properties: {}\n required: []\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 2); + assert(issues.every((i) => i.target === "script")); + }); +}); + +// --- Normal app inline script tests --- + +Deno.test("locks-required: fails for app with unlocked inline python script", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.app`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.app/app.yaml`, + `summary: My app +value: + grid: + - data: + inlineScript: + content: "def main(): pass" + language: python3 + lock: "" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "app_inline_script"); + assert(issues[0].errors[0].includes("python3")); + }); +}); + +Deno.test("locks-required: passes for app with locked inline python script", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.app`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.app/inline_script_0.inline_script.lock`, + `pandas==2.0.0\n`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.app/app.yaml`, + `summary: My app +value: + grid: + - data: + inlineScript: + content: "import pandas" + language: python3 + lock: "!inline inline_script_0.inline_script.lock" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: skips app inline bash scripts", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.app`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.app/app.yaml`, + `summary: My app +value: + grid: + - data: + inlineScript: + content: "echo hello" + language: bash + lock: "" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: finds deeply nested app inline scripts", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.app`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.app/app.yaml`, + `summary: My app +value: + grid: + - components: + - nested: + deeper: + inlineScript: + content: "export async function main() {}" + language: bun + lock: "" +`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "app_inline_script"); + assert(issues[0].errors[0].includes("bun")); + }); +}); + +// --- Raw app backend script tests --- + +Deno.test("locks-required: fails for raw app with unlocked backend python script", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.raw_app/backend`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/raw_app.yaml`, + `summary: My raw app +`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/get_data.yaml`, + `type: inline +`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/get_data.py`, + `def main(): pass`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "raw_app_inline_script"); + assert(issues[0].errors[0].includes("python3")); + assert(issues[0].errors[0].includes("get_data")); + }); +}); + +Deno.test("locks-required: passes for raw app with locked backend python script", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.raw_app/backend`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/raw_app.yaml`, + `summary: My raw app +`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/get_data.yaml`, + `type: inline +`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/get_data.py`, + `import pandas\ndef main(): pass`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/get_data.lock`, + `pandas==2.0.0\n`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +}); + +Deno.test("locks-required: raw app auto-detects code files without YAML config", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.raw_app/backend`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/raw_app.yaml`, + `summary: My raw app +`, + ); + // No .yaml config, just a code file + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/fetch_users.bun.ts`, + `export async function main() { return []; }`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 1); + assertEquals(issues[0].target, "raw_app_inline_script"); + assert(issues[0].errors[0].includes("bun")); + assert(issues[0].errors[0].includes("fetch_users")); + }); +}); + +Deno.test("locks-required: skips raw app bash backend scripts", async () => { + await withTempDir(async (tempDir) => { + await Deno.mkdir(`${tempDir}/f/my_app.raw_app/backend`, { recursive: true }); + + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/raw_app.yaml`, + `summary: My raw app +`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/cleanup.yaml`, + `type: inline +`, + ); + await Deno.writeTextFile( + `${tempDir}/f/my_app.raw_app/backend/cleanup.sh`, + `#!/bin/bash\necho done`, + ); + + const issues = await checkMissingLocks({} as any, tempDir); + assertEquals(issues.length, 0); + }); +});