diff --git a/cli/src/commands/app/app_metadata.ts b/cli/src/commands/app/app_metadata.ts index b4d37c0cba..25c3fccbd2 100644 --- a/cli/src/commands/app/app_metadata.ts +++ b/cli/src/commands/app/app_metadata.ts @@ -16,22 +16,30 @@ import { inferSchema, getRawWorkspaceDependencies, } from "../../utils/metadata.ts"; -import { ScriptLanguage, workspaceDependenciesLanguages } from "../../utils/script_common.ts"; import { - inferContentTypeFromFilePath, + ScriptLanguage, + workspaceDependenciesLanguages, } from "../../utils/script_common.ts"; +import { inferContentTypeFromFilePath } from "../../utils/script_common.ts"; import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; import { exts } from "../script/script.ts"; import { FSFSElement, yamlOptions } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; -import { AppFile } from "./raw_apps.ts"; -import { replaceInlineScripts } from "./apps.ts"; +import { AppFile as RawAppFile } from "./raw_apps.ts"; +import { replaceInlineScripts, AppFile as NormalAppFile } from "./apps.ts"; import { newPathAssigner, SupportedLanguage, } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts"; +import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; +import { resolveWorkspace } from "../../core/context.ts"; +import { requireLogin } from "../../core/auth.ts"; const TOP_HASH = "__app_hash"; +export const APP_BACKEND_FOLDER = "backend"; + +// Union type for app files that can be either raw or normal apps +type AppFile = RawAppFile | NormalAppFile; /** * Generates a hash for all inline scripts in an app directory @@ -39,27 +47,25 @@ const TOP_HASH = "__app_hash"; async function generateAppHash( rawReqs: Record | undefined, folder: string, + rawApp: boolean, defaultTs: "bun" | "deno" | undefined ): Promise> { - const runnablesFolder = path.join(folder, "runnables"); + const runnablesFolder = rawApp + ? path.join(folder, APP_BACKEND_FOLDER) + : folder; const hashes: Record = {}; try { const elems = await FSFSElement(runnablesFolder, [], true); for await (const f of elems.getChildren()) { + if (!rawApp && !f.path.includes(".inline_script.")) { + continue; + } if (exts.some((e) => f.path.endsWith(e))) { - let reqs: string | undefined; - if (rawReqs) { - // Get language name from path - const lang = inferContentTypeFromFilePath(f.path, defaultTs); - // Get lock for that language - [, reqs] = - Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? []; - } // Embed lock into hash const relativePath = f.path.replace(runnablesFolder + SEP, ""); hashes[relativePath] = await generateHash( - (await f.getContentText()) + (reqs ?? "") + (await f.getContentText()) + JSON.stringify(rawReqs) ); } } @@ -78,13 +84,14 @@ async function generateAppHash( */ export async function generateAppLocksInternal( appFolder: string, + rawApp: boolean, dryRun: boolean, workspace: Workspace, opts: GlobalOptions & { defaultTs?: "bun" | "deno"; }, justUpdateMetadataLock?: boolean, - noStaleMessage?: boolean, + noStaleMessage?: boolean ): Promise { if (appFolder.endsWith(SEP)) { appFolder = appFolder.substring(0, appFolder.length - 1); @@ -96,10 +103,15 @@ export async function generateAppLocksInternal( log.info(`Generating locks for app ${appFolder} at ${remote_path}`); } - const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); + const rawWorkspaceDependencies: Record = + await getRawWorkspaceDependencies(); - - let hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs); + let hashes = await generateAppHash( + rawWorkspaceDependencies, + appFolder, + rawApp, + opts.defaultTs + ); const conf = await import("../../utils/metadata.ts").then((m) => m.readLockfile() @@ -128,7 +140,10 @@ export async function generateAppLocksInternal( } // Read the app file - const appFilePath = path.join(appFolder, "raw_app.yaml"); + const appFilePath = path.join( + appFolder, + rawApp ? "raw_app.yaml" : "app.yaml" + ); const appFile = (await yamlParseFile(appFilePath)) as AppFile; if (!justUpdateMetadataLock) { @@ -148,20 +163,38 @@ export async function generateAppLocksInternal( `Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}` ); - const runnablesPath = path.join(appFolder, "runnables") + SEP; + if (rawApp) { + const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER) + SEP; + const rawAppFile = appFile as RawAppFile; - // Replace inline scripts for changed runnables - await replaceInlineScripts(appFile.runnables, runnablesPath); + // Replace inline scripts for changed runnables + replaceInlineScripts(rawAppFile.runnables, runnablesPath, false); - // Update the app runnables with new locks - appFile.runnables = await updateAppRunnables( - workspace, - appFile.runnables, - remote_path, - appFolder, - rawWorkspaceDependencies, - opts.defaultTs - ); + // Update the app runnables with new locks + rawAppFile.runnables = await updateRawAppRunnables( + workspace, + rawAppFile.runnables, + remote_path, + appFolder, + rawWorkspaceDependencies, + opts.defaultTs + ); + } else { + const normalAppFile = appFile as NormalAppFile; + + // Replace inline scripts for normal apps + replaceInlineScripts(normalAppFile.value, appFolder + SEP, false); + + // Update the app value with new locks + normalAppFile.value = await updateAppInlineScripts( + workspace, + normalAppFile.value, + remote_path, + appFolder, + rawWorkspaceDependencies, + opts.defaultTs + ); + } // Write the updated app file writeIfChanged( @@ -174,7 +207,12 @@ export async function generateAppLocksInternal( } // Regenerate hashes after updates - hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs); + hashes = await generateAppHash( + rawWorkspaceDependencies, + appFolder, + rawApp, + opts.defaultTs + ); await clearGlobalLock(appFolder); for (const [scriptPath, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(appFolder, hash, scriptPath); @@ -183,10 +221,68 @@ export async function generateAppLocksInternal( } /** - * Updates locks for all runnables in an app, generating locks inline script by inline script + * Callback type for processing inline scripts found during traversal + */ +type InlineScriptProcessor = ( + inlineScript: any, + context: { + path: string[]; + parentKey: string; + parentObject: any; + } +) => Promise; + +/** + * Traverses an app structure (either app.value for normal apps or app.runnables for raw apps) + * and processes all inline scripts found, returning the updated structure + */ +async function traverseAndProcessInlineScripts( + obj: any, + processor: InlineScriptProcessor, + currentPath: string[] = [] +): Promise { + if (!obj || typeof obj !== "object") { + return obj; + } + + if (Array.isArray(obj)) { + return await Promise.all( + obj.map((item, index) => + traverseAndProcessInlineScripts(item, processor, [ + ...currentPath, + `[${index}]`, + ]) + ) + ); + } + + const result: Record = {}; + + for (const [key, value] of Object.entries(obj)) { + if (key === "inlineScript" && typeof value === "object") { + // Found an inline script - process it + result[key] = await processor(value, { + path: currentPath, + parentKey: key, + parentObject: obj, + }); + } else { + // Recursively process nested objects + result[key] = await traverseAndProcessInlineScripts(value, processor, [ + ...currentPath, + key, + ]); + } + } + + return result; +} + +/** + * Updates locks for all runnables in a raw app, generating locks inline script by inline script * Also writes content and locks back to the runnables folder */ -async function updateAppRunnables( +async function updateRawAppRunnables( workspace: Workspace, runnables: Record, remotePath: string, @@ -194,8 +290,7 @@ async function updateAppRunnables( rawDeps?: Record, defaultTs: "bun" | "deno" = "bun" ): Promise> { - const updatedRunnables = { ...runnables }; - const runnablesFolder = path.join(appFolder, "runnables"); + const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER); // Ensure runnables folder exists try { @@ -205,9 +300,14 @@ async function updateAppRunnables( } const pathAssigner = newPathAssigner(defaultTs); + + // Process each runnable + const updatedRunnables: Record = {}; + for (const [runnableId, runnable] of Object.entries(runnables)) { // Only process inline scripts (runnableByName with inlineScript) if (runnable?.type !== "runnableByName" || !runnable?.inlineScript) { + updatedRunnables[runnableId] = runnable; continue; } @@ -216,6 +316,7 @@ async function updateAppRunnables( const content = inlineScript.content; if (!content || !language) { + updatedRunnables[runnableId] = runnable; continue; } @@ -226,21 +327,19 @@ async function updateAppRunnables( `Runnable ${runnableId} content is still an !inline reference, skipping` ) ); + updatedRunnables[runnableId] = runnable; continue; } // Skip frontend scripts - they don't need locks if (language === "frontend") { + updatedRunnables[runnableId] = runnable; continue; } - // Find raw deps for this language if available - const langRawDeps = rawDeps?.[language]; - log.info( colors.gray( - `Generating lock for runnable ${runnableId} (${language})${ - langRawDeps ? " with raw deps" : "" + `Generating lock for runnable ${runnableId} (${language}) }` ) ); @@ -251,7 +350,7 @@ async function updateAppRunnables( content, language, `${remotePath}/${runnableId}`, - langRawDeps + rawDeps ); // Determine file extension for this language @@ -294,12 +393,120 @@ async function updateAppRunnables( ) ); // Continue with other runnables even if one fails + updatedRunnables[runnableId] = runnable; } } return updatedRunnables; } +/** + * Updates locks for all inline scripts in a normal app, similar to updateRawAppRunnables + * but for the app.value structure instead of app.runnables + */ +async function updateAppInlineScripts( + workspace: Workspace, + appValue: any, + remotePath: string, + appFolder: string, + rawDeps?: Record, + defaultTs: "bun" | "deno" = "bun" +): Promise { + const pathAssigner = newPathAssigner(defaultTs); + + const processor: InlineScriptProcessor = async (inlineScript, context) => { + const language = inlineScript.language as SupportedLanguage; + const content = inlineScript.content; + + if (!content || !language) { + return inlineScript; + } + + // Skip if content is still an !inline reference (should have been replaced by replaceInlineScripts) + if (typeof content === "string" && content.startsWith("!inline ")) { + log.warn( + colors.yellow( + `Inline script at ${context.path.join( + "." + )} is still an !inline reference, skipping` + ) + ); + return inlineScript; + } + + // Skip frontend scripts - they don't need locks + if (language === "frontend") { + return inlineScript; + } + + // Get the name from the parent object (following extractInlineScriptsForApps pattern) + // For normal apps, the name is stored in the component's "name" property + const scriptName = context.parentObject?.["name"] || "unnamed"; + const scriptPath = `${remotePath}/${context.path.join("/")}`; + + log.info( + colors.gray( + `Generating lock for inline script "${scriptName}" at ${context.path.join( + "." + )} (${language})` + ) + ); + + try { + const lock = await generateInlineScriptLock( + workspace, + content, + language, + scriptPath, + rawDeps + ); + + // Determine file extension for this language (following extractInlineScriptsForApps pattern) + const [basePathO, ext] = pathAssigner.assignPath(scriptName, language); + const basePath = basePathO.replaceAll(SEP, "/"); + const contentPath = path.join(appFolder, `${basePath}${ext}`); + const lockPath = path.join(appFolder, `${basePath}lock`); + + // Write content to file + writeIfChanged(contentPath, content); + + // Write lock to file if it exists + if (lock && lock !== "") { + writeIfChanged(lockPath, lock); + } + + // Update the inline script with !inline references + const inlineContentRef = `!inline ${basePath}${ext}`; + const inlineLockRef = + lock && lock !== "" ? `!inline ${basePath}lock` : ""; + + log.info( + colors.gray( + ` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}` + ) + ); + + return { + ...inlineScript, + content: inlineContentRef, + lock: inlineLockRef, + }; + } catch (error: any) { + log.error( + colors.red( + `Failed to generate lock for inline script at ${context.path.join( + "." + )}: ${error.message}` + ) + ); + // Return original on error + return inlineScript; + } + }; + + return await traverseAndProcessInlineScripts(appValue, processor); +} + /** * Generates a lock for a single inline script using the dependencies endpoint */ @@ -308,7 +515,7 @@ async function generateInlineScriptLock( content: string, language: string, scriptPath: string, - rawDeps?: string + rawWorkspaceDependencies: Record | undefined ): Promise { const extraHeaders = getHeaders(); @@ -329,7 +536,11 @@ async function generateInlineScriptLock( script_path: scriptPath, }, ], - raw_deps: rawDeps, + raw_workspace_dependencies: + rawWorkspaceDependencies && + Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies + : null, entrypoint: scriptPath, }), } @@ -386,7 +597,7 @@ export interface InferredSchemaResult { */ export async function inferRunnableSchemaFromFile( appFolder: string, - runnableFilePath: string, + runnableFilePath: string ): Promise { // Extract runnable ID from file path (e.g., "myRunnable.inline_script.ts" -> "myRunnable") const fileName = path.basename(runnableFilePath); @@ -406,7 +617,7 @@ export async function inferRunnableSchemaFromFile( // Read the app file to get the language const appFilePath = path.join(appFolder, "raw_app.yaml"); - const appFile = (await yamlParseFile(appFilePath)) as AppFile; + const appFile = (await yamlParseFile(appFilePath)) as RawAppFile; if (!appFile.runnables?.[runnableId]) { log.warn(colors.yellow(`Runnable ${runnableId} not found in raw_app.yaml`)); @@ -423,10 +634,12 @@ export async function inferRunnableSchemaFromFile( const inlineScript = runnable.inlineScript; const language = inlineScript.language as SupportedLanguage; - - // Read the actual content from the file - const fullFilePath = path.join(appFolder, "runnables", runnableFilePath); + const fullFilePath = path.join( + appFolder, + APP_BACKEND_FOLDER, + runnableFilePath + ); let content: string; try { content = await Deno.readTextFile(fullFilePath); @@ -461,3 +674,144 @@ export async function inferRunnableSchemaFromFile( return undefined; } } + +function getAppFolders(elems: Record, extension: string) { + return Object.keys(elems) + .filter((p) => p.endsWith(SEP + extension)) + .map((p) => p.substring(0, p.length - (SEP + extension).length)); +} + +export async function generateLocksCommand( + opts: GlobalOptions & { + yes?: boolean; + dryRun?: boolean; + defaultTs?: "bun" | "deno"; + } & SyncOptions, + appPath: string | undefined +) { + const { generateAppLocksInternal } = await import("./app_metadata.ts"); + const { elementsToMap, FSFSElement } = await import("../sync/sync.ts"); + const { ignoreF } = await import("../sync/sync.ts"); + const { Confirm } = await import("../../../deps.ts"); + + if (appPath == "") { + appPath = undefined; + } + + const workspace = await resolveWorkspace(opts); + await requireLogin(opts); + opts = await mergeConfigWithConfigFile(opts); + + if (appPath) { + //TODO: Generate metadata for a specific raw app but handle normal apps to + throw new Error("Not implemented"); + // Generate metadata for a specific app + // await generateAppLocksInternal( + // appPath, + // true, + // false, + // workspace, + // opts, + // false, + // false + // ); + } else { + // Generate metadata for all apps + const ignore = await ignoreF(opts); + const elems = await elementsToMap( + await FSFSElement(Deno.cwd(), [], true), + (p, isD) => { + return ( + ignore(p, isD) || + (!isD && + !p.endsWith(SEP + "raw_app.yaml") && + !p.endsWith(SEP + "app.yaml")) + ); + }, + false, + {} + ); + + const rawAppFolders = getAppFolders(elems, "raw_app.yaml"); + const appFolders = getAppFolders(elems, "app.yaml"); + + let hasAny = false; + log.info( + `Checking metadata for all apps (${appFolders.length}) and raw apps (${rawAppFolders.length})` + ); + for (const appFolder of rawAppFolders) { + const candidate = await generateAppLocksInternal( + appFolder, + true, + true, + workspace, + opts, + false, + true + ); + if (candidate) { + hasAny = true; + log.info(colors.green(`+ ${candidate}`)); + } + } + + for (const appFolder of appFolders) { + const candidate = await generateAppLocksInternal( + appFolder, + false, + true, + workspace, + opts, + false, + true + ); + if (candidate) { + hasAny = true; + log.info(colors.green(`+ ${candidate}`)); + } + } + + if (hasAny) { + if (opts.dryRun) { + log.info(colors.gray(`Dry run complete.`)); + return; + } + if ( + !opts.yes && + !(await Confirm.prompt({ + message: "Update the metadata of the above apps?", + default: true, + })) + ) { + return; + } + } else { + log.info(colors.green.bold("No metadata to update")); + return; + } + + for (const appFolder of rawAppFolders) { + await generateAppLocksInternal( + appFolder, + true, + false, + workspace, + opts, + false, + true + ); + } + + for (const appFolder of appFolders) { + await generateAppLocksInternal( + appFolder, + false, + false, + workspace, + opts, + false, + true + ); + } + } +} diff --git a/cli/src/commands/app/apps.ts b/cli/src/commands/app/apps.ts index cb662b1670..e2b715b199 100644 --- a/cli/src/commands/app/apps.ts +++ b/cli/src/commands/app/apps.ts @@ -52,23 +52,31 @@ export function repopulateFields(runnables: Record) { } }); } -export function replaceInlineScripts(rec: any, localPath: string) { +export function replaceInlineScripts( + rec: any, + localPath: string, + addType: boolean +) { if (!rec) { return; } if (typeof rec == "object") { return Object.entries(rec).flatMap(([k, v]) => { if (k == "runType") { - if (isVersionsGeq1585()) { - rec["type"] = "path"; - } else { - rec["type"] = "runnableByPath"; + if (addType) { + if (isVersionsGeq1585()) { + rec["type"] = "path"; + } else { + rec["type"] = "runnableByPath"; + } } } else if (k == "inlineScript" && typeof v == "object") { - if (isVersionsGeq1585()) { - rec["type"] = "inline"; - } else { - rec["type"] = "runnableByName"; + if (addType) { + if (isVersionsGeq1585()) { + rec["type"] = "inline"; + } else { + rec["type"] = "runnableByName"; + } } const o: Record = v as any; @@ -81,7 +89,7 @@ export function replaceInlineScripts(rec: any, localPath: string) { o["lock"] = readInlinePathSync(basePath); } } else { - replaceInlineScripts(v, localPath); + replaceInlineScripts(v, localPath, addType); } }); } @@ -125,7 +133,7 @@ export async function pushApp( const path = localPath + "app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; - replaceInlineScripts(localApp.value, localPath); + replaceInlineScripts(localApp.value, localPath, true); await generatingPolicy( localApp, remotePath, @@ -234,7 +242,7 @@ const command = new Command() "Default TypeScript runtime (bun or deno)" ) .action(async (opts: any, appFolder: string | undefined) => { - const { generateLocksCommand } = await import("./raw_apps.ts"); + const { generateLocksCommand } = await import("./app_metadata.ts"); await generateLocksCommand(opts, appFolder); }); diff --git a/cli/src/commands/app/dev.ts b/cli/src/commands/app/dev.ts index ed8b565e10..327f06864e 100644 --- a/cli/src/commands/app/dev.ts +++ b/cli/src/commands/app/dev.ts @@ -7,6 +7,7 @@ import { open, windmillUtils, yamlParseFile, + SEP, } from "../../../deps.ts"; import { GlobalOptions } from "../../types.ts"; import * as http from "node:http"; @@ -16,7 +17,12 @@ import process from "node:process"; import { Buffer } from "node:buffer"; import { writeFileSync } from "node:fs"; import { WebSocketServer, WebSocket } from "npm:ws"; -import { getDevBuildOptions, ensureNodeModules, createFrameworkPlugins, detectFrameworks } from "./bundle.ts"; +import { + getDevBuildOptions, + ensureNodeModules, + createFrameworkPlugins, + detectFrameworks, +} from "./bundle.ts"; import { wmillTsDev as wmillTs } from "./wmillTsDev.ts"; import * as wmill from "../../../gen/services.gen.ts"; import { resolveWorkspace } from "../../core/context.ts"; @@ -24,7 +30,10 @@ import { requireLogin } from "../../core/auth.ts"; import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts"; import { replaceInlineScripts } from "./apps.ts"; import { Runnable } from "./metadata.ts"; -import { inferRunnableSchemaFromFile } from "./app_metadata.ts"; +import { + APP_BACKEND_FOLDER, + inferRunnableSchemaFromFile, +} from "./app_metadata.ts"; const DEFAULT_PORT = 4000; const DEFAULT_HOST = "localhost"; @@ -92,8 +101,8 @@ async function dev(opts: DevOptions) { log.error( colors.red( `Error: The dev command must be run inside a .raw_app folder.\n` + - `Current directory: ${currentDirName}\n` + - `Please navigate to a folder ending with '.raw_app' before running this command.` + `Current directory: ${currentDirName}\n` + + `Please navigate to a folder ending with '.raw_app' before running this command.` ) ); Deno.exit(1); @@ -105,7 +114,7 @@ async function dev(opts: DevOptions) { log.error( colors.red( `Error: raw_app.yaml not found in current directory.\n` + - `The dev command must be run in a .raw_app folder containing a raw_app.yaml file.` + `The dev command must be run in a .raw_app folder containing a raw_app.yaml file.` ) ); Deno.exit(1); @@ -133,7 +142,8 @@ async function dev(opts: DevOptions) { // Detect frameworks to determine default entry point const frameworks = detectFrameworks(process.cwd()); - const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const defaultEntry = + frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; const entryPoint = opts.entry ?? defaultEntry; // Verify entry point exists @@ -206,7 +216,6 @@ async function dev(opts: DevOptions) { }, }; - // Create esbuild context const ctx = await esbuild.context({ ...buildOptions, @@ -242,7 +251,7 @@ async function dev(opts: DevOptions) { await ctx.rebuild(); // Watch runnables folder for changes - const runnablesPath = path.join(process.cwd(), "runnables"); + const runnablesPath = path.join(process.cwd(), APP_BACKEND_FOLDER); let runnablesWatcher: Deno.FsWatcher | undefined; if (fs.existsSync(runnablesPath)) { @@ -262,7 +271,10 @@ async function dev(opts: DevOptions) { // Process each changed path with individual debouncing for (const changedPath of event.paths) { const relativePath = path.relative(process.cwd(), changedPath); - const relativeToRunnables = path.relative(runnablesPath, changedPath); + const relativeToRunnables = path.relative( + runnablesPath, + changedPath + ); // Skip non-modify events for schema inference if (event.kind !== "modify" && event.kind !== "create") { @@ -276,7 +288,9 @@ async function dev(opts: DevOptions) { // Log the change event log.info( - colors.cyan(`📝 Runnable changed [${event.kind}]: ${relativePath}`) + colors.cyan( + `📝 Runnable changed [${event.kind}]: ${relativePath}` + ) ); // Debounce schema inference per file (wait for typing to finish) @@ -288,7 +302,9 @@ async function dev(opts: DevOptions) { delete schemaInferenceTimeouts[changedPath]; try { - log.info(colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`)); + log.info( + colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`) + ); // Infer schema for this runnable (returns schema in memory, doesn't write to file) const result = await inferRunnableSchemaFromFile( process.cwd(), @@ -299,7 +315,15 @@ async function dev(opts: DevOptions) { // log.info(colors.green(` Runnable ID: ${result.runnableId}`)); // Store inferred schema in memory inferredSchemas[result.runnableId] = result.schema; - log.info(colors.green(` Inferred Schemas: ${JSON.stringify(inferredSchemas, null, 2)}`)); + log.info( + colors.green( + ` Inferred Schemas: ${JSON.stringify( + inferredSchemas, + null, + 2 + )}` + ) + ); // Regenerate wmill.d.ts with updated schema from memory await genRunnablesTs(inferredSchemas); } @@ -468,7 +492,9 @@ async function dev(opts: DevOptions) { case "backendAsync": { // Run a runnable asynchronously and return job ID immediately log.info( - colors.blue(`[backendAsync] Running runnable async: ${runnable_id}`) + colors.blue( + `[backendAsync] Running runnable async: ${runnable_id}` + ) ); try { const runnables = await loadRunnables(); @@ -628,7 +654,10 @@ const command = new Command() .option("--host ", "Host to bind the dev server to", { default: DEFAULT_HOST, }) - .option("--entry ", "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)") + .option( + "--entry ", + "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)" + ) .option("--no-open", "Don't automatically open the browser") .action(dev as any); @@ -671,7 +700,11 @@ async function loadRunnables(): Promise> { const rawApp = (await yamlParseFile( path.join(localPath, "raw_app.yaml") )) as any; - replaceInlineScripts(rawApp.runnables, path.join(localPath, "runnables/")); + replaceInlineScripts( + rawApp.runnables, + path.join(localPath, APP_BACKEND_FOLDER) + SEP, + true + ); return rawApp?.runnables ?? {}; } catch (error: any) { @@ -707,7 +740,10 @@ async function executeRunnable( } } - if ((runnable.type === "inline" || runnable.type === "runnableByName") && runnable.inlineScript) { + if ( + (runnable.type === "inline" || runnable.type === "runnableByName") && + runnable.inlineScript + ) { const inlineScript = runnable.inlineScript; if (inlineScript.id !== undefined) { requestBody.id = inlineScript.id; @@ -719,7 +755,10 @@ async function executeRunnable( lock: inlineScript.id === undefined ? inlineScript.lock : undefined, cache_ttl: inlineScript.cache_ttl, }; - } else if ((runnable.type === "path" || runnable.type === "runnableByPath") && runnable.path) { + } else if ( + (runnable.type === "path" || runnable.type === "runnableByPath") && + runnable.path + ) { const runType = runnable.runType ?? "script"; requestBody.path = runType !== "hubscript" diff --git a/cli/src/commands/app/raw_apps.ts b/cli/src/commands/app/raw_apps.ts index cb2d42f05b..200b697024 100644 --- a/cli/src/commands/app/raw_apps.ts +++ b/cli/src/commands/app/raw_apps.ts @@ -15,7 +15,7 @@ import { GlobalOptions, isSuperset } from "../../types.ts"; import { replaceInlineScripts, repopulateFields } from "./apps.ts"; import { createBundle, detectFrameworks } from "./bundle.ts"; -import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts"; +import { APP_BACKEND_FOLDER } from "./app_metadata.ts"; export interface AppFile { runnables: any; @@ -39,7 +39,12 @@ async function collectAppFiles( if (entry.isDirectory) { // Skip the runnables and node_modules subfolders - if (entry.name === "runnables" || entry.name === "node_modules" || entry.name === "dist" || entry.name === ".claude") { + if ( + entry.name === APP_BACKEND_FOLDER || + entry.name === "node_modules" || + entry.name === "dist" || + entry.name === ".claude" + ) { continue; } await readDirRecursive(fullPath + SEP, relativePath + SEP); @@ -96,15 +101,20 @@ export async function pushRawApp( } const path = localPath + "raw_app.yaml"; const localApp = (await yamlParseFile(path)) as AppFile; - replaceInlineScripts(localApp.runnables, localPath + SEP + "runnables/"); - repopulateFields(localApp.runnables) + replaceInlineScripts( + localApp.runnables, + localPath + SEP + APP_BACKEND_FOLDER + SEP, + true + ); + repopulateFields(localApp.runnables); await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false); const files = await collectAppFiles(localPath); async function createBundleRaw() { log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`)); // Detect frameworks to determine entry point const frameworks = detectFrameworks(localPath); - const entryFile = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx"; + const entryFile = + frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx"; const entryPoint = localPath + entryFile; return await createBundle({ entryPoint: entryPoint, @@ -183,102 +193,6 @@ export async function generatingPolicy( } } -export async function generateLocksCommand( - opts: GlobalOptions & { - yes?: boolean; - dryRun?: boolean; - defaultTs?: "bun" | "deno"; - } & SyncOptions, - appPath: string | undefined -) { - const { generateAppLocksInternal } = await import("./app_metadata.ts"); - const { elementsToMap, FSFSElement } = await import("../sync/sync.ts"); - const { ignoreF } = await import("../sync/sync.ts"); - const { Confirm } = await import("../../../deps.ts"); - - if (appPath == "") { - appPath = undefined; - } - - const workspace = await resolveWorkspace(opts); - await requireLogin(opts); - opts = await mergeConfigWithConfigFile(opts); - - if (appPath) { - // Generate metadata for a specific app - await generateAppLocksInternal( - appPath, - false, - workspace, - opts, - false, - false, - ); - } else { - // Generate metadata for all apps - const ignore = await ignoreF(opts); - const elems = await elementsToMap( - await FSFSElement(Deno.cwd(), [], true), - (p, isD) => { - return ignore(p, isD) || (!isD && !p.endsWith(SEP + "raw_app.yaml")); - }, - false, - {} - ); - - const appFolders = Object.keys(elems) - .filter((p) => p.endsWith(SEP + "raw_app.yaml")) - .map((p) => p.substring(0, p.length - (SEP + "raw_app.yaml").length)); - - let hasAny = false; - log.info("Checking metadata for all apps:"); - for (const appFolder of appFolders) { - const candidate = await generateAppLocksInternal( - appFolder, - true, - workspace, - opts, - false, - true, - ); - if (candidate) { - hasAny = true; - log.info(colors.green(`+ ${candidate}`)); - } - } - - if (hasAny) { - if (opts.dryRun) { - log.info(colors.gray(`Dry run complete.`)); - return; - } - if ( - !opts.yes && - !(await Confirm.prompt({ - message: "Update the metadata of the above apps?", - default: true, - })) - ) { - return; - } - } else { - log.info(colors.green.bold("No metadata to update")); - return; - } - - for (const appFolder of appFolders) { - await generateAppLocksInternal( - appFolder, - false, - workspace, - opts, - false, - true, - ); - } - } -} - async function pushRawAppCommand( opts: GlobalOptions, filePath: string, diff --git a/cli/src/commands/flow/flow_metadata.ts b/cli/src/commands/flow/flow_metadata.ts index fb5c5539da..2eadf1b705 100644 --- a/cli/src/commands/flow/flow_metadata.ts +++ b/cli/src/commands/flow/flow_metadata.ts @@ -18,9 +18,8 @@ import { } from "../../utils/metadata.ts"; import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; - import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts"; -import { exts, } from "../script/script.ts"; +import { exts } from "../script/script.ts"; import { FSFSElement } from "../sync/sync.ts"; import { Workspace } from "../workspace/workspace.ts"; import { FlowFile } from "./flow.ts"; @@ -67,8 +66,13 @@ export async function generateFlowLockInternal( } // Always get out-of-sync workspace dependencies - const rawWorkspaceDependencies: Record = await getRawWorkspaceDependencies(); - let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs); + const rawWorkspaceDependencies: Record = + await getRawWorkspaceDependencies(); + let hashes = await generateFlowHash( + rawWorkspaceDependencies, + folder, + opts.defaultTs + ); const conf = await readLockfile(); if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { @@ -115,7 +119,7 @@ export async function generateFlowLockInternal( log, folder + SEP!, SEP, - changedScripts, + changedScripts // (path: string, newPath: string) => Deno.renameSync(path, newPath), // (path: string) => Deno.removeSync(path) ); @@ -145,7 +149,11 @@ export async function generateFlowLockInternal( ); } - hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs); + hashes = await generateFlowHash( + rawWorkspaceDependencies, + folder, + opts.defaultTs + ); await clearGlobalLock(folder); for (const [path, hash] of Object.entries(hashes)) { await updateMetadataGlobalLock(folder, hash, path); @@ -153,8 +161,6 @@ export async function generateFlowLockInternal( log.info(colors.green(`Flow ${remote_path} lockfiles updated`)); } - - export async function updateFlow( workspace: Workspace, flow_value: FlowValue, @@ -164,7 +170,9 @@ export async function updateFlow( let rawResponse; if (Object.keys(rawWorkspaceDependencies).length > 0) { - log.info(colors.blue("Using raw workspace dependencies for flow dependencies")); + log.info( + colors.blue("Using raw workspace dependencies for flow dependencies") + ); // generate the script lock running a dependency job in Windmill and update it inplace const extraHeaders = getHeaders(); @@ -181,9 +189,10 @@ export async function updateFlow( flow_value, path: remotePath, use_local_lockfiles: true, - raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0 - ? rawWorkspaceDependencies - : null, + raw_workspace_dependencies: + Object.keys(rawWorkspaceDependencies).length > 0 + ? rawWorkspaceDependencies + : null, }), } ); diff --git a/cli/src/commands/sync/sync.ts b/cli/src/commands/sync/sync.ts index 8fffa5ed32..57222f482b 100644 --- a/cli/src/commands/sync/sync.ts +++ b/cli/src/commands/sync/sync.ts @@ -36,6 +36,7 @@ import { import { handleFile } from "../script/script.ts"; import { deepEqual, + fetchRemoteVersion, isFileResource, isRawAppFile, isWorkspaceDependencies, @@ -74,8 +75,10 @@ import { import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts"; import { generateFlowLockInternal } from "../flow/flow_metadata.ts"; import { isExecutionModeAnonymous } from "../app/apps.ts"; -import { generateAppLocksInternal } from "../app/app_metadata.ts"; -import { updateGlobalVersions } from "./global.ts"; +import { + APP_BACKEND_FOLDER, + generateAppLocksInternal, +} from "../app/app_metadata.ts"; // Merge CLI options with effective settings, preserving CLI flags as overrides function mergeCliWithEffectiveOptions< @@ -549,7 +552,7 @@ function ZipFSElement( for (const s of inlineScripts) { yield { isDirectory: false, - path: path.join(finalPath, "runnables", s.path), + path: path.join(finalPath, APP_BACKEND_FOLDER, s.path), async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { @@ -1737,6 +1740,19 @@ export async function pull( await generateAppLocksInternal( change, false, + true, + workspace, + opts, + true, + true + ); + } + for (const change of tracker.apps) { + log.info(`Updating lock metadata for app ${change}`); + await generateAppLocksInternal( + change, + false, + false, workspace, opts, true, @@ -1983,6 +1999,7 @@ export async function push( const staleScripts: string[] = []; const staleFlows: string[] = []; + const staleApps: string[] = []; for (const change of tracker.scripts) { const stale = await generateScriptMetadataInternal( @@ -2036,16 +2053,51 @@ export async function push( log.info(""); } - const version = await fetchVersion(workspace.remote); - if (version) { - updateGlobalVersions(version); + for (const change of tracker.apps) { + const stale = await generateAppLocksInternal( + change, + false, + true, + workspace, + opts, + true, + true + ); + if (stale) { + staleApps.push(stale); + } } - log.info(colors.gray("Remote version: " + version)); + + for (const change of tracker.rawApps) { + const stale = await generateAppLocksInternal( + change, + true, + true, + workspace, + opts, + true, + true + ); + if (stale) { + staleApps.push(stale); + } + } + + if (staleApps.length > 0) { + log.warn( + "Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:" + ); + for (const stale of staleApps) { + log.warn(stale); + } + log.info(""); + } + + await fetchRemoteVersion(workspace); log.info( `remote (${workspace.name}) <- local: ${changes.length} changes to apply` ); - // Handle JSON output for dry-run if (opts.dryRun && opts.jsonOutput) { const result = { diff --git a/cli/src/main.ts b/cli/src/main.ts index cd3853f819..2f8d5e9b17 100644 --- a/cli/src/main.ts +++ b/cli/src/main.ts @@ -25,7 +25,6 @@ import instance from "./commands/instance/instance.ts"; import workerGroups from "./commands/worker-groups/worker_groups.ts"; import dev from "./commands/dev/dev.ts"; -import { fetchVersion } from "./core/context.ts"; import { GlobalOptions } from "./types.ts"; import { OpenAPI } from "../gen/index.ts"; import { getHeaders, getIsWin } from "./utils/utils.ts"; @@ -39,6 +38,7 @@ import queues from "./commands/queues/queues.ts"; import dependencies from "./commands/dependencies/dependencies.ts"; import init from "./commands/init/init.ts"; import jobs from "./commands/jobs/jobs.ts"; +import { fetchVersion } from "./core/context.ts"; export { flow, diff --git a/cli/src/utils/utils.ts b/cli/src/utils/utils.ts index bda8dea78a..ceee3d9d7a 100644 --- a/cli/src/utils/utils.ts +++ b/cli/src/utils/utils.ts @@ -4,6 +4,8 @@ import { colors, encodeHex, log, SEP } from "../../deps.ts"; import crypto from "node:crypto"; +import { fetchVersion } from "../core/context.ts"; +import { updateGlobalVersions } from "../commands/sync/global.ts"; export function deepEqual(a: T, b: T): boolean { if (a === b) return true; @@ -151,11 +153,11 @@ export function isFileResource(path: string): boolean { } export function isRawAppFile(path: string): boolean { - return path.includes(".raw_app" + SEP) ; + return path.includes(".raw_app" + SEP); } export function isWorkspaceDependencies(path: string): boolean { - return path.startsWith("dependencies/") + return path.startsWith("dependencies/"); } export function printSync(input: string | Uint8Array, to = Deno.stdout) { @@ -263,3 +265,13 @@ export function writeIfChanged(path: string, content: string): boolean { Deno.writeTextFileSync(path, content); return true; // File was written } + +export async function fetchRemoteVersion( + workspace: Workspace +): Promise { + const version = await fetchVersion(workspace.remote); + if (version) { + updateGlobalVersions(version); + } + log.info(colors.gray("Remote version: " + version)); +}