From ef3e2353a76d096847b3f10b1daa6767fc4baa0d Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Thu, 31 Jul 2025 14:23:38 +0000 Subject: [PATCH] feat(cli): wmill-lock.yaml v2 for easier git merge diffs --- cli/deno.lock | 10 + cli/main.ts | 741 +++++++++--------- cli/metadata.ts | 66 +- cli/sync.ts | 119 ++- .../src/inline-scripts/extractor.ts | 40 +- 5 files changed, 529 insertions(+), 447 deletions(-) diff --git a/cli/deno.lock b/cli/deno.lock index 497c0b7ac2..801b233835 100644 --- a/cli/deno.lock +++ b/cli/deno.lock @@ -59,6 +59,7 @@ "jsr:@windmill-labs/cliffy-table@1.0.0-rc.5": "1.0.0-rc.5", "jsr:@windmill-labs/cliffy-table@^1.0.0-rc.5": "1.0.0-rc.5", "npm:@ayonli/jsext@*": "1.8.0", + "npm:@types/node@*": "22.12.0", "npm:centdix-utils@*": "1.0.15", "npm:diff@*": "8.0.2", "npm:es-main@*": "1.3.0", @@ -378,6 +379,12 @@ "@isaacs/balanced-match" ] }, + "@types/node@22.12.0": { + "integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==", + "dependencies": [ + "undici-types" + ] + }, "accepts@2.0.0": { "integrity": "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==", "dependencies": [ @@ -905,6 +912,9 @@ "mime-types" ] }, + "undici-types@6.20.0": { + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" + }, "unpipe@1.0.0": { "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" }, diff --git a/cli/main.ts b/cli/main.ts index ef3bd67809..0469846e68 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -1,11 +1,11 @@ import { - Command, - CompletionsCommand, - UpgradeCommand, - colors, - esMain, - log, - yamlStringify, + Command, + CompletionsCommand, + UpgradeCommand, + colors, + esMain, + log, + yamlStringify, } from "./deps.ts"; import flow from "./flow.ts"; import app from "./apps.ts"; @@ -40,26 +40,26 @@ import { readLockfile } from "./metadata.ts"; import { FLOW_GUIDANCE } from "./flow_guidance.ts"; export { - flow, - app, - script, - workspace, - resource, - resourceType, - user, - variable, - hub, - folder, - schedule, - trigger, - sync, - gitsyncSettings, - instance, - dev, - hubPull, - pull, - push, - workspaceAdd, + flow, + app, + script, + workspace, + resource, + resourceType, + user, + variable, + hub, + folder, + schedule, + trigger, + sync, + gitsyncSettings, + instance, + dev, + hubPull, + pull, + push, + workspaceAdd, }; // addEventListener("error", (event) => { @@ -72,230 +72,226 @@ export { export const VERSION = "1.516.0"; const command = new Command() - .name("wmill") - .action(() => - log.info(`Welcome to Windmill CLI ${VERSION}. Use -h for help.`), - ) - .description("Windmill CLI") + .name("wmill") + .action(() => + log.info(`Welcome to Windmill CLI ${VERSION}. Use -h for help.`) + ) + .description("Windmill CLI") - .globalOption( - "--workspace ", - "Specify the target workspace. This overrides the default workspace.", - ) - .globalOption("--debug --verbose", "Show debug/verbose logs") - .globalOption( - "--show-diffs", - "Show diff informations when syncing (may show sensitive informations)", - ) - .globalOption( - "--token ", - "Specify an API token. This will override any stored token.", - ) - .globalOption( - "--base-url ", - "Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used.", - ) - .globalOption( - "--config-dir ", - "Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location.", - ) - .env( - "HEADERS ", - "Specify headers to use for all requests. e.g: \"HEADERS='h1: v1, h2: v2'\"", - ) - .version(VERSION) - .versionOption(false) - .command("init", "Bootstrap a windmill project with a wmill.yaml file") - .option("--use-default", "Use default settings without checking backend") - .option("--use-backend", "Use backend git-sync settings if available") - .option( - "--repository ", - "Specify repository path (e.g., u/user/repo) when using backend settings", - ) - .action( - async ( - opts: { - useDefault?: boolean; - useBackend?: boolean; - repository?: string; - workspace?: string; - debug?: unknown; - showDiffs?: boolean; - token?: string; - baseUrl?: string; - configDir?: string; - }, - ) => { - if (await Deno.stat("wmill.yaml").catch(() => null)) { - log.error(colors.red("wmill.yaml already exists")); - } else { - // Import DEFAULT_SYNC_OPTIONS from conf.ts - const { DEFAULT_SYNC_OPTIONS } = await import("./conf.ts"); + .globalOption( + "--workspace ", + "Specify the target workspace. This overrides the default workspace." + ) + .globalOption("--debug --verbose", "Show debug/verbose logs") + .globalOption( + "--show-diffs", + "Show diff informations when syncing (may show sensitive informations)" + ) + .globalOption( + "--token ", + "Specify an API token. This will override any stored token." + ) + .globalOption( + "--base-url ", + "Specify the base URL of the API. If used, --token and --workspace are required and no local remote/workspace already set will be used." + ) + .globalOption( + "--config-dir ", + "Specify a custom config directory. Overrides WMILL_CONFIG_DIR environment variable and default ~/.config location." + ) + .env( + "HEADERS ", + "Specify headers to use for all requests. e.g: \"HEADERS='h1: v1, h2: v2'\"" + ) + .version(VERSION) + .versionOption(false) + .command("init", "Bootstrap a windmill project with a wmill.yaml file") + .option("--use-default", "Use default settings without checking backend") + .option("--use-backend", "Use backend git-sync settings if available") + .option( + "--repository ", + "Specify repository path (e.g., u/user/repo) when using backend settings" + ) + .action( + async (opts: { + useDefault?: boolean; + useBackend?: boolean; + repository?: string; + workspace?: string; + debug?: unknown; + showDiffs?: boolean; + token?: string; + baseUrl?: string; + configDir?: string; + }) => { + if (await Deno.stat("wmill.yaml").catch(() => null)) { + log.error(colors.red("wmill.yaml already exists")); + } else { + // Import DEFAULT_SYNC_OPTIONS from conf.ts + const { DEFAULT_SYNC_OPTIONS } = await import("./conf.ts"); - // Create initial config with defaults - const initialConfig = { - defaultTs: DEFAULT_SYNC_OPTIONS.defaultTs, - includes: DEFAULT_SYNC_OPTIONS.includes, - excludes: DEFAULT_SYNC_OPTIONS.excludes, - codebases: DEFAULT_SYNC_OPTIONS.codebases, - skipVariables: DEFAULT_SYNC_OPTIONS.skipVariables, - skipResources: DEFAULT_SYNC_OPTIONS.skipResources, - skipSecrets: DEFAULT_SYNC_OPTIONS.skipSecrets, - skipScripts: DEFAULT_SYNC_OPTIONS.skipScripts, - skipFlows: DEFAULT_SYNC_OPTIONS.skipFlows, - skipApps: DEFAULT_SYNC_OPTIONS.skipApps, - skipFolders: DEFAULT_SYNC_OPTIONS.skipFolders, - includeSchedules: DEFAULT_SYNC_OPTIONS.includeSchedules, - includeTriggers: DEFAULT_SYNC_OPTIONS.includeTriggers, - overrides: {}, - }; + // Create initial config with defaults + const initialConfig = { + defaultTs: DEFAULT_SYNC_OPTIONS.defaultTs, + includes: DEFAULT_SYNC_OPTIONS.includes, + excludes: DEFAULT_SYNC_OPTIONS.excludes, + codebases: DEFAULT_SYNC_OPTIONS.codebases, + skipVariables: DEFAULT_SYNC_OPTIONS.skipVariables, + skipResources: DEFAULT_SYNC_OPTIONS.skipResources, + skipSecrets: DEFAULT_SYNC_OPTIONS.skipSecrets, + skipScripts: DEFAULT_SYNC_OPTIONS.skipScripts, + skipFlows: DEFAULT_SYNC_OPTIONS.skipFlows, + skipApps: DEFAULT_SYNC_OPTIONS.skipApps, + skipFolders: DEFAULT_SYNC_OPTIONS.skipFolders, + includeSchedules: DEFAULT_SYNC_OPTIONS.includeSchedules, + includeTriggers: DEFAULT_SYNC_OPTIONS.includeTriggers, + overrides: {}, + }; - await Deno.writeTextFile( - "wmill.yaml", - yamlStringify(initialConfig), - ); - log.info( - colors.green("wmill.yaml created with default settings"), - ); + await Deno.writeTextFile("wmill.yaml", yamlStringify(initialConfig)); + log.info(colors.green("wmill.yaml created with default settings")); - // Create lock file - await readLockfile(); + // Create lock file + await readLockfile(); - // Check for backend git-sync settings unless --use-default is specified - if (!opts.useDefault) { - try { - const { requireLogin } = await import("./auth.ts"); - const { resolveWorkspace } = await import("./context.ts"); + // Check for backend git-sync settings unless --use-default is specified + if (!opts.useDefault) { + try { + const { requireLogin } = await import("./auth.ts"); + const { resolveWorkspace } = await import("./context.ts"); - // Check if user has workspace configured - const { getActiveWorkspace } = await import( - "./workspace.ts" - ); - const activeWorkspace = await getActiveWorkspace(opts as GlobalOptions); + // Check if user has workspace configured + const { getActiveWorkspace } = await import("./workspace.ts"); + const activeWorkspace = await getActiveWorkspace( + opts as GlobalOptions + ); - if (!activeWorkspace) { - log.info( - "No workspace configured. Using default settings.", - ); - log.info( - "You can configure a workspace later with 'wmill workspace add'", - ); - return; - } - - await requireLogin(opts as GlobalOptions); - const workspace = await resolveWorkspace(opts as GlobalOptions); - - const wmill = await import("./gen/services.gen.ts"); - const settings = await wmill.getSettings({ - workspace: workspace.workspaceId, - }); - - if ( - settings.git_sync?.repositories && - settings.git_sync.repositories.length > 0 - ) { - let useBackendSettings = opts.useBackend; - - // If repository is specified, implicitly use backend settings - if (opts.repository && !opts.useDefault) { - useBackendSettings = true; - } - - if (useBackendSettings === undefined) { - // Interactive prompt - const { Select } = await import("./deps.ts"); - const choice = await Select.prompt({ - message: - "Git-sync settings found on backend. What would you like to do?", - options: [ - { - name: "Use backend git-sync settings", - value: "backend", - }, - { - name: "Use default settings", - value: "default", - }, - { - name: "Cancel", - value: "cancel", - }, - ], - }); - - if (choice === "cancel") { - // Clean up the created files - try { - await Deno.remove("wmill.yaml"); - await Deno.remove("wmill-lock.yaml"); - } catch (e) { - // Ignore cleanup errors - } - log.info("Init cancelled"); - Deno.exit(0); - } - - useBackendSettings = choice === "backend"; - } - - if (useBackendSettings) { - log.info( - "Applying git-sync settings from backend...", - ); - - // Import and run the pull git-sync settings logic - const { pullGitSyncSettings } = await import( - "./gitsync-settings.ts" - ); - await pullGitSyncSettings({ - ...(opts as GlobalOptions), - repository: opts.repository, - jsonOutput: false, - diff: false, - replace: true, // Auto-replace when using backend settings during init - }); - - log.info( - colors.green( - "Git-sync settings applied from backend", - ), - ); - } - } - } catch (error) { - // If there's an error checking backend settings, just continue with defaults - const errorMessage = error instanceof Error ? error.message : String(error); - log.warn( - `Could not check backend for git-sync settings: ${errorMessage}`, - ); - log.info("Continuing with default settings"); - } - } + if (!activeWorkspace) { + log.info("No workspace configured. Using default settings."); + log.info( + "You can configure a workspace later with 'wmill workspace add'" + ); + return; } - // Create .cursor/rules directory and files with SCRIPT_GUIDANCE content - try { - const scriptGuidanceContent = SCRIPT_GUIDANCE; - const flowGuidanceContent = FLOW_GUIDANCE; - - // Create .cursor/rules directory - await Deno.mkdir(".cursor/rules", { recursive: true }); - - // Create windmill.mdc file - if (!await Deno.stat(".cursor/rules/script.mdc").catch(() => null)) { - await Deno.writeTextFile(".cursor/rules/script.mdc", scriptGuidanceContent); - log.info(colors.green("Created .cursor/rules/script.mdc")); + await requireLogin(opts as GlobalOptions); + const workspace = await resolveWorkspace(opts as GlobalOptions); + + const wmill = await import("./gen/services.gen.ts"); + const settings = await wmill.getSettings({ + workspace: workspace.workspaceId, + }); + + if ( + settings.git_sync?.repositories && + settings.git_sync.repositories.length > 0 + ) { + let useBackendSettings = opts.useBackend; + + // If repository is specified, implicitly use backend settings + if (opts.repository && !opts.useDefault) { + useBackendSettings = true; + } + + if (useBackendSettings === undefined) { + // Interactive prompt + const { Select } = await import("./deps.ts"); + const choice = await Select.prompt({ + message: + "Git-sync settings found on backend. What would you like to do?", + options: [ + { + name: "Use backend git-sync settings", + value: "backend", + }, + { + name: "Use default settings", + value: "default", + }, + { + name: "Cancel", + value: "cancel", + }, + ], + }); + + if (choice === "cancel") { + // Clean up the created files + try { + await Deno.remove("wmill.yaml"); + await Deno.remove("wmill-lock.yaml"); + } catch (e) { + // Ignore cleanup errors + } + log.info("Init cancelled"); + Deno.exit(0); } - if (!await Deno.stat(".cursor/rules/flow.mdc").catch(() => null)) { - await Deno.writeTextFile(".cursor/rules/flow.mdc", flowGuidanceContent); - log.info(colors.green("Created .cursor/rules/flow.mdc")); - } - - // Create CLAUDE.md file - if (!await Deno.stat("CLAUDE.md").catch(() => null)) { - await Deno.writeTextFile("CLAUDE.md", ` + useBackendSettings = choice === "backend"; + } + + if (useBackendSettings) { + log.info("Applying git-sync settings from backend..."); + + // Import and run the pull git-sync settings logic + const { pullGitSyncSettings } = await import( + "./gitsync-settings.ts" + ); + await pullGitSyncSettings({ + ...(opts as GlobalOptions), + repository: opts.repository, + jsonOutput: false, + diff: false, + replace: true, // Auto-replace when using backend settings during init + }); + + log.info( + colors.green("Git-sync settings applied from backend") + ); + } + } + } catch (error) { + // If there's an error checking backend settings, just continue with defaults + const errorMessage = + error instanceof Error ? error.message : String(error); + log.warn( + `Could not check backend for git-sync settings: ${errorMessage}` + ); + log.info("Continuing with default settings"); + } + } + } + + // Create .cursor/rules directory and files with SCRIPT_GUIDANCE content + try { + const scriptGuidanceContent = SCRIPT_GUIDANCE; + const flowGuidanceContent = FLOW_GUIDANCE; + + // Create .cursor/rules directory + await Deno.mkdir(".cursor/rules", { recursive: true }); + + // Create windmill.mdc file + if (!(await Deno.stat(".cursor/rules/script.mdc").catch(() => null))) { + await Deno.writeTextFile( + ".cursor/rules/script.mdc", + scriptGuidanceContent + ); + log.info(colors.green("Created .cursor/rules/script.mdc")); + } + + if (!(await Deno.stat(".cursor/rules/flow.mdc").catch(() => null))) { + await Deno.writeTextFile( + ".cursor/rules/flow.mdc", + flowGuidanceContent + ); + log.info(colors.green("Created .cursor/rules/flow.mdc")); + } + + // Create CLAUDE.md file + if (!(await Deno.stat("CLAUDE.md").catch(() => null))) { + await Deno.writeTextFile( + "CLAUDE.md", + ` # Claude You are a helpful assistant that can help with Windmill scripts and flows creation. @@ -305,162 +301,161 @@ const command = new Command() ## Flow Guidance ${flowGuidanceContent} - `); - log.info(colors.green("Created CLAUDE.md")); - } - - } catch (error) { - if (error instanceof Error) { - log.warn(`Could not create guidance files: ${error.message}`); - } else { - log.warn(`Could not create guidance files: ${error}`); - } - } - - }, - ) - .command("app", app) - .command("flow", flow) - .command("script", script) - .command("workspace", workspace) - .command("resource", resource) - .command("resource-type", resourceType) - .command("user", user) - .command("variable", variable) - .command("hub", hub) - .command("folder", folder) - .command("schedule", schedule) - .command("trigger", trigger) - .command("dev", dev) - .command("sync", sync) - .command("gitsync-settings", gitsyncSettings) - .command("instance", instance) - .command("worker-groups", workerGroups) - .command("workers", workers) - .command("queues", queues) - .command("version --version", "Show version information") - .action(async (opts) => { - console.log("CLI version: " + VERSION); - try { - const provider = new NpmProvider({ package: "windmill-cli" }); - const versions = await provider.getVersions("windmill-cli"); - if (versions.latest !== VERSION) { - console.log( - `CLI is outdated. Latest version ${versions.latest} is available. Run \`wmill upgrade\` to update.`, - ); - } else { - console.log("CLI is up to date"); - } - } catch (e) { - console.warn( - `Cannot fetch latest CLI version on npmjs to check if up-to-date: ${e}`, - ); + ` + ); + log.info(colors.green("Created CLAUDE.md")); } - const workspace = await getActiveWorkspace(opts as GlobalOptions); - if (workspace) { - try { - const backendVersion = await fetchVersion(workspace.remote); - console.log("Backend Version: " + backendVersion); - } catch (e) { - console.warn("Cannot fetch backend version: " + e); - } + } catch (error) { + if (error instanceof Error) { + log.warn(`Could not create guidance files: ${error.message}`); } else { - console.warn( - "Cannot fetch backend version: no active workspace selected, choose one to pick a remote to fetch version of", - ); + log.warn(`Could not create guidance files: ${error}`); } + } + } + ) + .command("app", app) + .command("flow", flow) + .command("script", script) + .command("workspace", workspace) + .command("resource", resource) + .command("resource-type", resourceType) + .command("user", user) + .command("variable", variable) + .command("hub", hub) + .command("folder", folder) + .command("schedule", schedule) + .command("trigger", trigger) + .command("dev", dev) + .command("sync", sync) + .command("gitsync-settings", gitsyncSettings) + .command("instance", instance) + .command("worker-groups", workerGroups) + .command("workers", workers) + .command("queues", queues) + .command("version --version", "Show version information") + .action(async (opts) => { + console.log("CLI version: " + VERSION); + try { + const provider = new NpmProvider({ package: "windmill-cli" }); + const versions = await provider.getVersions("windmill-cli"); + if (versions.latest !== VERSION) { + console.log( + `CLI is outdated. Latest version ${versions.latest} is available. Run \`wmill upgrade\` to update.` + ); + } else { + console.log("CLI is up to date"); + } + } catch (e) { + console.warn( + `Cannot fetch latest CLI version on npmjs to check if up-to-date: ${e}` + ); + } + const workspace = await getActiveWorkspace(opts as GlobalOptions); + if (workspace) { + try { + const backendVersion = await fetchVersion(workspace.remote); + console.log("Backend Version: " + backendVersion); + } catch (e) { + console.warn("Cannot fetch backend version: " + e); + } + } else { + console.warn( + "Cannot fetch backend version: no active workspace selected, choose one to pick a remote to fetch version of" + ); + } + }) + .command( + "upgrade", + new UpgradeCommand({ + provider: new NpmProvider({ package: "windmill-cli" }), + }).error((e: any) => { + log.error(e); + log.info( + "Try running with sudo and otherwise check the result of the command: npm uninstall windmill-cli && npm install -g windmill-cli" + ); }) - .command( - "upgrade", - new UpgradeCommand({ - provider: new NpmProvider({ package: "windmill-cli" }), - }).error((e: any) => { - log.error(e); - log.info( - "Try running with sudo and otherwise check the result of the command: npm uninstall windmill-cli && npm install -g windmill-cli", - ); - }), - ) - .command("completions", new CompletionsCommand()); + ) + .command("completions", new CompletionsCommand()); export let showDiffs = false; let isWin: boolean | undefined = undefined; export async function getIsWin() { - if (isWin === undefined) { - const os = await import("node:os"); - isWin = os.platform() === "win32"; - } - return isWin; + if (isWin === undefined) { + const os = await import("node:os"); + isWin = os.platform() === "win32"; + } + return isWin; } async function main() { - try { - if (Deno.args.length === 0) { - command.showHelp(); - } - const LOG_LEVEL = - Deno.args.includes("--verbose") || Deno.args.includes("--debug") - ? "DEBUG" - : "INFO"; - // const NO_COLORS = Deno.args.includes("--no-colors"); - showDiffs = Deno.args.includes("--show-diffs"); - - log.setup({ - handlers: { - console: new log.ConsoleHandler(LOG_LEVEL, { - formatter: ({ msg }) => `${msg}`, - useColors: isWin ? false : true, - }), - }, - loggers: { - default: { - level: LOG_LEVEL, - handlers: ["console"], - }, - }, - }); - log.debug("Debug logging enabled. CLI build against " + VERSION); - - const extraHeaders = getHeaders(); - if (extraHeaders) { - OpenAPI.HEADERS = extraHeaders; - } - await command.parse(Deno.args); - } catch (e) { - if (e.name === "ApiError") { - console.log("Server failed. " + e.statusText + ": " + e.body); - } - throw e; + try { + if (Deno.args.length === 0) { + command.showHelp(); } + const LOG_LEVEL = + Deno.args.includes("--verbose") || Deno.args.includes("--debug") + ? "DEBUG" + : "INFO"; + // const NO_COLORS = Deno.args.includes("--no-colors"); + showDiffs = Deno.args.includes("--show-diffs"); + + log.setup({ + handlers: { + console: new log.ConsoleHandler(LOG_LEVEL, { + formatter: ({ msg }) => `${msg}`, + useColors: isWin ? false : true, + }), + }, + loggers: { + default: { + level: LOG_LEVEL, + handlers: ["console"], + }, + }, + }); + log.debug("Debug logging enabled. CLI build against " + VERSION); + + const extraHeaders = getHeaders(); + if (extraHeaders) { + OpenAPI.HEADERS = extraHeaders; + } + await command.parse(Deno.args); + } catch (e) { + if (e.name === "ApiError") { + console.log("Server failed. " + e.statusText + ": " + e.body); + } + throw e; + } } function isMain() { - // dnt-shim-ignore - const { Deno } = globalThis as any; + // dnt-shim-ignore + const { Deno } = globalThis as any; - const isDeno = Deno != undefined; + const isDeno = Deno != undefined; - if (isDeno) { - const isMain = import.meta.main; - if (isMain) { - if (!Deno.args.includes("completions")) { - if (Deno.env.get("SKIP_DENO_DEPRECATION_WARNING") !== "true") { - log.warn( - "Using the deno runtime for the Windmill CLI is deprecated, you can now use node: deno uninstall wmill && npm install -g windmill-cli. To skip this warning set SKIP_DENO_DEPRECATION_WARNING=true", - ); - } - } + if (isDeno) { + const isMain = import.meta.main; + if (isMain) { + if (!Deno.args.includes("completions")) { + if (Deno.env.get("SKIP_DENO_DEPRECATION_WARNING") !== "true") { + log.warn( + "Using the deno runtime for the Windmill CLI is deprecated, you can now use node: deno uninstall wmill && npm install -g windmill-cli. To skip this warning set SKIP_DENO_DEPRECATION_WARNING=true" + ); } - return isMain; - } else { - //@ts-ignore - return esMain.default(import.meta); + } } + return isMain; + } else { + //@ts-ignore + return esMain.default(import.meta); + } } if (isMain()) { - main(); + main(); } export default command; diff --git a/cli/metadata.ts b/cli/metadata.ts index 75de6e3989..8dc63a3026 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -20,11 +20,7 @@ import { } from "./script_common.ts"; import { inferContentTypeFromFilePath } from "./script_common.ts"; import { GlobalDeps, exts, findGlobalDeps } from "./script.ts"; -import { - FSFSElement, - findCodebase, - yamlOptions, -} from "./sync.ts"; +import { FSFSElement, findCodebase, yamlOptions } from "./sync.ts"; import { generateHash, readInlinePathSync } from "./utils.ts"; import { SyncCodebase } from "./codebase.ts"; import { FlowFile } from "./flow.ts"; @@ -180,7 +176,7 @@ export async function generateFlowLockInternal( SEP, changedScripts, (path: string, newPath: string) => Deno.renameSync(path, newPath), - (path: string) => Deno.removeSync(path), + (path: string) => Deno.removeSync(path) ); //removeChangedLocks @@ -191,7 +187,11 @@ export async function generateFlowLockInternal( rawReqs ); - const inlineScripts = extractInlineScriptsForFlows(flowValue.value.modules, {}, SEP); + const inlineScripts = extractInlineScriptsForFlows( + flowValue.value.modules, + {}, + SEP + ); inlineScripts .filter((s) => s.path.endsWith(".lock")) .forEach((s) => { @@ -531,9 +531,7 @@ export async function inferSchema( }> { let inferedSchema: any; if (language === "python3") { - const { parse_python } = await import( - "./wasm/py/windmill_parser_wasm.js" - ); + const { parse_python } = await import("./wasm/py/windmill_parser_wasm.js"); inferedSchema = JSON.parse(parse_python(content)); } else if (language === "nativets") { const { parse_deno } = await import("./wasm/ts/windmill_parser_wasm.js"); @@ -601,7 +599,9 @@ export async function inferSchema( ...inferedSchema.args, ]; } else if (language === "duckdb") { - const { parse_duckdb } = await import("./wasm/regex/windmill_parser_wasm.js"); + const { parse_duckdb } = await import( + "./wasm/regex/windmill_parser_wasm.js" + ); inferedSchema = JSON.parse(parse_duckdb(content)); } else if (language === "graphql") { const { parse_graphql } = await import( @@ -816,6 +816,7 @@ export async function parseMetadataFile( } interface Lock { + version?: "v2"; locks?: { [path: string]: string | { [subpath: string]: string } }; } @@ -829,7 +830,7 @@ export async function readLockfile(): Promise { throw new Error("Invalid lockfile"); } } catch { - const lock = { locks: {} }; + const lock = { locks: {}, version: "v2" as const }; await Deno.writeTextFile(WMILL_LOCKFILE, yamlStringify(lock, yamlOptions)); log.info(colors.green("wmill-lock.yaml created")); @@ -837,6 +838,13 @@ export async function readLockfile(): Promise { } } +function v2LockPath(path: string, subpath?: string) { + if (subpath) { + return `${path}+${subpath}`; + } else { + return path; + } +} export async function checkifMetadataUptodate( path: string, hash: string, @@ -849,9 +857,16 @@ export async function checkifMetadataUptodate( if (!conf.locks) { return false; } - const obj = conf.locks?.[path]; - const current = subpath && typeof obj == "object" ? obj?.[subpath] : obj; - return current == hash; + const isV2 = conf?.version == "v2"; + + if (isV2) { + const current = conf.locks?.[v2LockPath(path, subpath)]; + return current == hash; + } else { + const obj = conf.locks?.[path]; + const current = subpath && typeof obj == "object" ? obj?.[subpath] : obj; + return current == hash; + } } export async function generateScriptHash( @@ -873,16 +888,21 @@ export async function updateMetadataGlobalLock( if (!conf?.locks) { conf.locks = {}; } + const isV2 = conf?.version == "v2"; - if (subpath) { - let prev: any = conf.locks[path]; - if (!prev || typeof prev != "object") { - prev = {}; - conf.locks[path] = prev; - } - prev[subpath] = hash; + if (isV2) { + conf.locks[v2LockPath(path, hash)] = hash; } else { - conf.locks[path] = hash; + if (subpath) { + let prev: any = conf.locks[path]; + if (!prev || typeof prev != "object") { + prev = {}; + conf.locks[path] = prev; + } + prev[subpath] = hash; + } else { + conf.locks[path] = hash; + } } await Deno.writeTextFile( WMILL_LOCKFILE, diff --git a/cli/sync.ts b/cli/sync.ts index d64040e9ff..9f5b919434 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -52,10 +52,9 @@ import { assignPath } from "./windmill-utils-internal/src/path-utils/path-assign import { extractInlineScripts as extractInlineScriptsForFlows } from "./windmill-utils-internal/src/inline-scripts/extractor.ts"; // Merge CLI options with effective settings, preserving CLI flags as overrides -function mergeCliWithEffectiveOptions( - cliOpts: T, - effectiveOpts: SyncOptions -): T { +function mergeCliWithEffectiveOptions< + T extends GlobalOptions & SyncOptions & { repository?: string } +>(cliOpts: T, effectiveOpts: SyncOptions): T { // overlay CLI options on top (undefined cliOpts won't override effectiveOpts) return Object.assign({}, effectiveOpts, cliOpts) as T; } @@ -84,7 +83,7 @@ async function resolveEffectiveSyncOptions( // Find all repository-specific overrides for this workspace for (const key of Object.keys(localConfig.overrides)) { - if (key.startsWith(prefix) && !key.endsWith(':*')) { + if (key.startsWith(prefix) && !key.endsWith(":*")) { const repo = key.substring(prefix.length); if (repo) { applicableRepos.push(repo); @@ -107,13 +106,16 @@ async function resolveEffectiveSyncOptions( if (isInteractive) { const choices = [ - { name: "Use top-level settings (no repository-specific override)", value: "" }, - ...applicableRepos.map(repo => ({ name: repo, value: repo })) + { + name: "Use top-level settings (no repository-specific override)", + value: "", + }, + ...applicableRepos.map((repo) => ({ name: repo, value: repo })), ]; const selectedRepo = await Select.prompt({ message: "Multiple repository overrides found. Select which to use:", - options: choices + options: choices, }); if (selectedRepo) { @@ -128,9 +130,15 @@ async function resolveEffectiveSyncOptions( ); } else { // Non-interactive mode - list options and use top-level - log.warn(`Multiple repository overrides found: ${applicableRepos.join(', ')}`); - log.warn(`Running in non-interactive mode. Use --repository flag to specify which one to use.`); - log.info(`Falling back to top-level settings (no repository-specific overrides applied)`); + log.warn( + `Multiple repository overrides found: ${applicableRepos.join(", ")}` + ); + log.warn( + `Running in non-interactive mode. Use --repository flag to specify which one to use.` + ); + log.info( + `Falling back to top-level settings (no repository-specific overrides applied)` + ); } } } @@ -280,6 +288,7 @@ export async function FSFSElement( } function prioritizeName(name: string): string { + if (name == "version") return "aaa"; if (name == "id") return "aa"; if (name == "type") return "ab"; if (name == "summary") return "ad"; @@ -292,6 +301,7 @@ function prioritizeName(name: string): string { if (name == "failure_module") return "ak"; if (name == "input_transforms") return "al"; if (name == "lock") return "az"; + if (name == "locks") return "azz"; return name; } @@ -310,9 +320,7 @@ export interface InlineScript { content: string; } -export function extractInlineScriptsForApps( - rec: any, -): InlineScript[] { +export function extractInlineScriptsForApps(rec: any): InlineScript[] { if (!rec) { return []; } @@ -390,7 +398,11 @@ function ZipFSElement( async *getChildren(): AsyncIterable { if (kind == "flow") { const flow: OpenFlow = JSON.parse(await f.async("text")); - const inlineScripts = extractInlineScriptsForFlows(flow.value.modules, {}, SEP); + const inlineScripts = extractInlineScriptsForFlows( + flow.value.modules, + {}, + SEP + ); for (const s of inlineScripts) { yield { isDirectory: false, @@ -1020,7 +1032,8 @@ export async function ignoreF(wmillconf: { wmillconf.includes?.some((i) => minimatch(file, i))) && (!wmillconf?.excludes || wmillconf.excludes!.every((i) => !minimatch(file, i))) && - (!wmillconf.extraIncludes || wmillconf.extraIncludes.length === 0 || + (!wmillconf.extraIncludes || + wmillconf.extraIncludes.length === 0 || wmillconf.extraIncludes.some((i) => minimatch(file, i))) ); }, @@ -1122,7 +1135,9 @@ async function buildTracker(changes: Change[]) { return tracker; } -export async function pull(opts: GlobalOptions & SyncOptions & { repository?: string }) { +export async function pull( + opts: GlobalOptions & SyncOptions & { repository?: string } +) { if (opts.stateful) { await ensureDir(path.join(Deno.cwd(), ".wmill")); } @@ -1131,7 +1146,10 @@ export async function pull(opts: GlobalOptions & SyncOptions & { repository?: st await requireLogin(opts); // Resolve effective sync options with repository awareness - const effectiveOpts = await resolveEffectiveSyncOptions(workspace, opts.repository); + const effectiveOpts = await resolveEffectiveSyncOptions( + workspace, + opts.repository + ); // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(opts, effectiveOpts); @@ -1195,12 +1213,14 @@ export async function pull(opts: GlobalOptions & SyncOptions & { repository?: st if (opts.dryRun && opts.jsonOutput) { const result = { success: true, - changes: changes.map(change => ({ + changes: changes.map((change) => ({ type: change.name, path: change.path, - ...(change.name === "edited" && change.codebase ? { codebase_changed: true } : {}) + ...(change.name === "edited" && change.codebase + ? { codebase_changed: true } + : {}), })), - total: changes.length + total: changes.length, }; console.log(JSON.stringify(result, null, 2)); return; @@ -1361,12 +1381,14 @@ export async function pull(opts: GlobalOptions & SyncOptions & { repository?: st const result = { success: true, message: `All ${changes.length} changes applied locally and wmill-lock.yaml updated`, - changes: changes.map(change => ({ + changes: changes.map((change) => ({ type: change.name, path: change.path, - ...(change.name === "edited" && change.codebase ? { codebase_changed: true } : {}) + ...(change.name === "edited" && change.codebase + ? { codebase_changed: true } + : {}), })), - total: changes.length + total: changes.length, }; console.log(JSON.stringify(result, null, 2)); } else { @@ -1377,7 +1399,13 @@ export async function pull(opts: GlobalOptions & SyncOptions & { repository?: st ); } } else if (opts.jsonOutput) { - console.log(JSON.stringify({ success: true, message: "No changes to apply", total: 0 }, null, 2)); + console.log( + JSON.stringify( + { success: true, message: "No changes to apply", total: 0 }, + null, + 2 + ) + ); } } @@ -1432,12 +1460,17 @@ function removeSuffix(str: string, suffix: string) { return str.slice(0, str.length - suffix.length); } -export async function push(opts: GlobalOptions & SyncOptions & { repository?: string }) { +export async function push( + opts: GlobalOptions & SyncOptions & { repository?: string } +) { const workspace = await resolveWorkspace(opts); await requireLogin(opts); // Resolve effective sync options with repository awareness - const effectiveOpts = await resolveEffectiveSyncOptions(workspace, opts.repository); + const effectiveOpts = await resolveEffectiveSyncOptions( + workspace, + opts.repository + ); // Merge CLI flags with resolved settings (CLI flags take precedence only for explicit overrides) opts = mergeCliWithEffectiveOptions(opts, effectiveOpts); @@ -1574,12 +1607,14 @@ export async function push(opts: GlobalOptions & SyncOptions & { repository?: st if (opts.dryRun && opts.jsonOutput) { const result = { success: true, - changes: changes.map(change => ({ + changes: changes.map((change) => ({ type: change.name, path: change.path, - ...(change.name === "edited" && change.codebase ? { codebase_changed: true } : {}) + ...(change.name === "edited" && change.codebase + ? { codebase_changed: true } + : {}), })), - total: changes.length + total: changes.length, }; console.log(JSON.stringify(result, null, 2)); return; @@ -1953,26 +1988,38 @@ export async function push(opts: GlobalOptions & SyncOptions & { repository?: st const result = { success: true, message: `All ${changes.length} changes pushed to the remote workspace ${workspace.workspaceId} named ${workspace.name}`, - changes: changes.map(change => ({ + changes: changes.map((change) => ({ type: change.name, path: change.path, - ...(change.name === "edited" && change.codebase ? { codebase_changed: true } : {}) + ...(change.name === "edited" && change.codebase + ? { codebase_changed: true } + : {}), })), total: changes.length, - duration_ms: Math.round(performance.now() - start) + duration_ms: Math.round(performance.now() - start), }; console.log(JSON.stringify(result, null, 2)); } else { log.info( colors.bold.green.underline( - `\nDone! All ${changes.length} changes pushed to the remote workspace ${ + `\nDone! All ${ + changes.length + } changes pushed to the remote workspace ${ workspace.workspaceId - } named ${workspace.name} (${(performance.now() - start).toFixed(0)}ms)` + } named ${workspace.name} (${(performance.now() - start).toFixed( + 0 + )}ms)` ) ); } } else if (opts.jsonOutput) { - console.log(JSON.stringify({ success: true, message: "No changes to push", total: 0 }, null, 2)); + console.log( + JSON.stringify( + { success: true, message: "No changes to push", total: 0 }, + null, + 2 + ) + ); } } diff --git a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts index 2262090e65..515caf8163 100644 --- a/cli/windmill-utils-internal/src/inline-scripts/extractor.ts +++ b/cli/windmill-utils-internal/src/inline-scripts/extractor.ts @@ -14,7 +14,7 @@ interface InlineScript { /** * Extracts inline scripts from flow modules, converting them to separate files * and replacing the original content with file references. - * + * * @param modules - Array of flow modules to process * @param mapping - Optional mapping of module IDs to custom file paths * @param defaultTs - Default TypeScript runtime to use ("bun" or "deno") @@ -28,11 +28,7 @@ export function extractInlineScripts( ): InlineScript[] { return modules.flatMap((m) => { if (m.value.type == "rawscript") { - const [basePath, ext] = assignPath( - m.id, - m.value.language, - defaultTs - ); + const [basePath, ext] = assignPath(m.id, m.value.language, defaultTs); const path = mapping[m.id] ?? basePath + ext; const content = m.value.content; const r = [{ path: path, content: content }]; @@ -45,13 +41,23 @@ export function extractInlineScripts( } return r; } else if (m.value.type == "forloopflow") { - return extractInlineScripts(m.value.modules, mapping, separator, defaultTs); + return extractInlineScripts( + m.value.modules, + mapping, + separator, + defaultTs + ); } else if (m.value.type == "branchall") { return m.value.branches.flatMap((b) => extractInlineScripts(b.modules, mapping, separator, defaultTs) ); } else if (m.value.type == "whileloopflow") { - return extractInlineScripts(m.value.modules, mapping, separator, defaultTs); + return extractInlineScripts( + m.value.modules, + mapping, + separator, + defaultTs + ); } else if (m.value.type == "branchone") { return [ ...m.value.branches.flatMap((b) => @@ -68,7 +74,7 @@ export function extractInlineScripts( /** * Extracts the current mapping of module IDs to file paths from flow modules * by analyzing existing inline script references. - * + * * @param modules - Array of flow modules to analyze (can be undefined) * @param mapping - Existing mapping to extend (defaults to empty object) * @returns Record mapping module IDs to their corresponding file paths @@ -80,12 +86,12 @@ export function extractCurrentMapping( if (!modules || !Array.isArray(modules)) { return mapping; } - + modules.forEach((m) => { if (!m?.value?.type) { return; } - + if (m.value.type === "rawscript") { if (m.value.content && m.value.content.startsWith("!inline ")) { mapping[m.id] = m.value.content.trim().split(" ")[1]; @@ -96,12 +102,16 @@ export function extractCurrentMapping( ) { extractCurrentMapping(m.value.modules, mapping); } else if (m.value.type === "branchall") { - m.value.branches.forEach((b) => extractCurrentMapping(b.modules, mapping)); + m.value.branches.forEach((b) => + extractCurrentMapping(b.modules, mapping) + ); } else if (m.value.type === "branchone") { - m.value.branches.forEach((b) => extractCurrentMapping(b.modules, mapping)); + m.value.branches.forEach((b) => + extractCurrentMapping(b.modules, mapping) + ); extractCurrentMapping(m.value.default, mapping); } }); - + return mapping; -} \ No newline at end of file +}