starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-21 08:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: wmill app generate-locks (#7288)

Browse Source 
* all

* fix(cli): wmill app generate-locks

* handle raw reqs

* handle raw reqs

* rework paths
This commit is contained in:
Ruben Fiszel
2025-12-03 20:14:01 +01:00
committed by GitHub
parent adfb7e809a
commit 99b1ae2e03
8 changed files with 590 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/src/commands/app/app_metadata.ts
+403 -49
View File
@@ -16,22 +16,30 @@ import {
inferSchema,
getRawWorkspaceDependencies,
} from "../../utils/metadata.ts";
import { ScriptLanguage, workspaceDependenciesLanguages } from "../../utils/script_common.ts";
import {
inferContentTypeFromFilePath,
ScriptLanguage,
workspaceDependenciesLanguages,
} from "../../utils/script_common.ts";
import { inferContentTypeFromFilePath } from "../../utils/script_common.ts";
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
import { exts } from "../script/script.ts";
import { FSFSElement, yamlOptions } from "../sync/sync.ts";
import { Workspace } from "../workspace/workspace.ts";
import { AppFile } from "./raw_apps.ts";
import { replaceInlineScripts } from "./apps.ts";
import { AppFile as RawAppFile } from "./raw_apps.ts";
import { replaceInlineScripts, AppFile as NormalAppFile } from "./apps.ts";
import {
newPathAssigner,
SupportedLanguage,
} from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
const TOP_HASH = "__app_hash";
export const APP_BACKEND_FOLDER = "backend";
// Union type for app files that can be either raw or normal apps
type AppFile = RawAppFile | NormalAppFile;
/**
* Generates a hash for all inline scripts in an app directory
@@ -39,27 +47,25 @@ const TOP_HASH = "__app_hash";
async function generateAppHash(
rawReqs: Record<string, string> | undefined,
folder: string,
rawApp: boolean,
defaultTs: "bun" | "deno" | undefined
): Promise<Record<string, string>> {
const runnablesFolder = path.join(folder, "runnables");
const runnablesFolder = rawApp
? path.join(folder, APP_BACKEND_FOLDER)
: folder;
const hashes: Record<string, string> = {};
try {
const elems = await FSFSElement(runnablesFolder, [], true);
for await (const f of elems.getChildren()) {
if (!rawApp && !f.path.includes(".inline_script.")) {
continue;
}
if (exts.some((e) => f.path.endsWith(e))) {
let reqs: string | undefined;
if (rawReqs) {
// Get language name from path
const lang = inferContentTypeFromFilePath(f.path, defaultTs);
// Get lock for that language
[, reqs] =
Object.entries(rawReqs).find(([lang2, _]) => lang == lang2) ?? [];
}
// Embed lock into hash
const relativePath = f.path.replace(runnablesFolder + SEP, "");
hashes[relativePath] = await generateHash(
(await f.getContentText()) + (reqs ?? "")
(await f.getContentText()) + JSON.stringify(rawReqs)
);
}
}
@@ -78,13 +84,14 @@ async function generateAppHash(
*/
export async function generateAppLocksInternal(
appFolder: string,
rawApp: boolean,
dryRun: boolean,
workspace: Workspace,
opts: GlobalOptions & {
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean,
noStaleMessage?: boolean
): Promise<string | void> {
if (appFolder.endsWith(SEP)) {
appFolder = appFolder.substring(0, appFolder.length - 1);
@@ -96,10 +103,15 @@ export async function generateAppLocksInternal(
log.info(`Generating locks for app ${appFolder} at ${remote_path}`);
}
const rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
const rawWorkspaceDependencies: Record<string, string> =
await getRawWorkspaceDependencies();
let hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs);
let hashes = await generateAppHash(
rawWorkspaceDependencies,
appFolder,
rawApp,
opts.defaultTs
);
const conf = await import("../../utils/metadata.ts").then((m) =>
m.readLockfile()
@@ -128,7 +140,10 @@ export async function generateAppLocksInternal(
}
// Read the app file
const appFilePath = path.join(appFolder, "raw_app.yaml");
const appFilePath = path.join(
appFolder,
rawApp ? "raw_app.yaml" : "app.yaml"
);
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
if (!justUpdateMetadataLock) {
@@ -148,20 +163,38 @@ export async function generateAppLocksInternal(
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
);
const runnablesPath = path.join(appFolder, "runnables") + SEP;
if (rawApp) {
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER) + SEP;
const rawAppFile = appFile as RawAppFile;
// Replace inline scripts for changed runnables
await replaceInlineScripts(appFile.runnables, runnablesPath);
// Replace inline scripts for changed runnables
replaceInlineScripts(rawAppFile.runnables, runnablesPath, false);
// Update the app runnables with new locks
appFile.runnables = await updateAppRunnables(
workspace,
appFile.runnables,
remote_path,
appFolder,
rawWorkspaceDependencies,
opts.defaultTs
);
// Update the app runnables with new locks
rawAppFile.runnables = await updateRawAppRunnables(
workspace,
rawAppFile.runnables,
remote_path,
appFolder,
rawWorkspaceDependencies,
opts.defaultTs
);
} else {
const normalAppFile = appFile as NormalAppFile;
// Replace inline scripts for normal apps
replaceInlineScripts(normalAppFile.value, appFolder + SEP, false);
// Update the app value with new locks
normalAppFile.value = await updateAppInlineScripts(
workspace,
normalAppFile.value,
remote_path,
appFolder,
rawWorkspaceDependencies,
opts.defaultTs
);
}
// Write the updated app file
writeIfChanged(
@@ -174,7 +207,12 @@ export async function generateAppLocksInternal(
}
// Regenerate hashes after updates
hashes = await generateAppHash(rawWorkspaceDependencies, appFolder, opts.defaultTs);
hashes = await generateAppHash(
rawWorkspaceDependencies,
appFolder,
rawApp,
opts.defaultTs
);
await clearGlobalLock(appFolder);
for (const [scriptPath, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(appFolder, hash, scriptPath);
@@ -183,10 +221,68 @@ export async function generateAppLocksInternal(
}
/**
* Updates locks for all runnables in an app, generating locks inline script by inline script
* Callback type for processing inline scripts found during traversal
*/
type InlineScriptProcessor = (
inlineScript: any,
context: {
path: string[];
parentKey: string;
parentObject: any;
}
) => Promise<any>;
/**
* Traverses an app structure (either app.value for normal apps or app.runnables for raw apps)
* and processes all inline scripts found, returning the updated structure
*/
async function traverseAndProcessInlineScripts(
obj: any,
processor: InlineScriptProcessor,
currentPath: string[] = []
): Promise<any> {
if (!obj || typeof obj !== "object") {
return obj;
}
if (Array.isArray(obj)) {
return await Promise.all(
obj.map((item, index) =>
traverseAndProcessInlineScripts(item, processor, [
...currentPath,
`[${index}]`,
])
)
);
}
const result: Record<string, any> = {};
for (const [key, value] of Object.entries(obj)) {
if (key === "inlineScript" && typeof value === "object") {
// Found an inline script - process it
result[key] = await processor(value, {
path: currentPath,
parentKey: key,
parentObject: obj,
});
} else {
// Recursively process nested objects
result[key] = await traverseAndProcessInlineScripts(value, processor, [
...currentPath,
key,
]);
}
}
return result;
}
/**
* Updates locks for all runnables in a raw app, generating locks inline script by inline script
* Also writes content and locks back to the runnables folder
*/
async function updateAppRunnables(
async function updateRawAppRunnables(
workspace: Workspace,
runnables: Record<string, any>,
remotePath: string,
@@ -194,8 +290,7 @@ async function updateAppRunnables(
rawDeps?: Record<string, string>,
defaultTs: "bun" | "deno" = "bun"
): Promise<Record<string, any>> {
const updatedRunnables = { ...runnables };
const runnablesFolder = path.join(appFolder, "runnables");
const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER);
// Ensure runnables folder exists
try {
@@ -205,9 +300,14 @@ async function updateAppRunnables(
}
const pathAssigner = newPathAssigner(defaultTs);
// Process each runnable
const updatedRunnables: Record<string, any> = {};
for (const [runnableId, runnable] of Object.entries(runnables)) {
// Only process inline scripts (runnableByName with inlineScript)
if (runnable?.type !== "runnableByName" || !runnable?.inlineScript) {
updatedRunnables[runnableId] = runnable;
continue;
}
@@ -216,6 +316,7 @@ async function updateAppRunnables(
const content = inlineScript.content;
if (!content || !language) {
updatedRunnables[runnableId] = runnable;
continue;
}
@@ -226,21 +327,19 @@ async function updateAppRunnables(
`Runnable ${runnableId} content is still an !inline reference, skipping`
)
);
updatedRunnables[runnableId] = runnable;
continue;
}
// Skip frontend scripts - they don't need locks
if (language === "frontend") {
updatedRunnables[runnableId] = runnable;
continue;
}
// Find raw deps for this language if available
const langRawDeps = rawDeps?.[language];
log.info(
colors.gray(
`Generating lock for runnable ${runnableId} (${language})${
langRawDeps ? " with raw deps" : ""
`Generating lock for runnable ${runnableId} (${language})
}`
)
);
@@ -251,7 +350,7 @@ async function updateAppRunnables(
content,
language,
`${remotePath}/${runnableId}`,
langRawDeps
rawDeps
);
// Determine file extension for this language
@@ -294,12 +393,120 @@ async function updateAppRunnables(
)
);
// Continue with other runnables even if one fails
updatedRunnables[runnableId] = runnable;
}
}
return updatedRunnables;
}
/**
* Updates locks for all inline scripts in a normal app, similar to updateRawAppRunnables
* but for the app.value structure instead of app.runnables
*/
async function updateAppInlineScripts(
workspace: Workspace,
appValue: any,
remotePath: string,
appFolder: string,
rawDeps?: Record<string, string>,
defaultTs: "bun" | "deno" = "bun"
): Promise<any> {
const pathAssigner = newPathAssigner(defaultTs);
const processor: InlineScriptProcessor = async (inlineScript, context) => {
const language = inlineScript.language as SupportedLanguage;
const content = inlineScript.content;
if (!content || !language) {
return inlineScript;
}
// Skip if content is still an !inline reference (should have been replaced by replaceInlineScripts)
if (typeof content === "string" && content.startsWith("!inline ")) {
log.warn(
colors.yellow(
`Inline script at ${context.path.join(
"."
)} is still an !inline reference, skipping`
)
);
return inlineScript;
}
// Skip frontend scripts - they don't need locks
if (language === "frontend") {
return inlineScript;
}
// Get the name from the parent object (following extractInlineScriptsForApps pattern)
// For normal apps, the name is stored in the component's "name" property
const scriptName = context.parentObject?.["name"] || "unnamed";
const scriptPath = `${remotePath}/${context.path.join("/")}`;
log.info(
colors.gray(
`Generating lock for inline script "${scriptName}" at ${context.path.join(
"."
)} (${language})`
)
);
try {
const lock = await generateInlineScriptLock(
workspace,
content,
language,
scriptPath,
rawDeps
);
// Determine file extension for this language (following extractInlineScriptsForApps pattern)
const [basePathO, ext] = pathAssigner.assignPath(scriptName, language);
const basePath = basePathO.replaceAll(SEP, "/");
const contentPath = path.join(appFolder, `${basePath}${ext}`);
const lockPath = path.join(appFolder, `${basePath}lock`);
// Write content to file
writeIfChanged(contentPath, content);
// Write lock to file if it exists
if (lock && lock !== "") {
writeIfChanged(lockPath, lock);
}
// Update the inline script with !inline references
const inlineContentRef = `!inline ${basePath}${ext}`;
const inlineLockRef =
lock && lock !== "" ? `!inline ${basePath}lock` : "";
log.info(
colors.gray(
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
)
);
return {
...inlineScript,
content: inlineContentRef,
lock: inlineLockRef,
};
} catch (error: any) {
log.error(
colors.red(
`Failed to generate lock for inline script at ${context.path.join(
"."
)}: ${error.message}`
)
);
// Return original on error
return inlineScript;
}
};
return await traverseAndProcessInlineScripts(appValue, processor);
}
/**
* Generates a lock for a single inline script using the dependencies endpoint
*/
@@ -308,7 +515,7 @@ async function generateInlineScriptLock(
content: string,
language: string,
scriptPath: string,
rawDeps?: string
rawWorkspaceDependencies: Record<string, string> | undefined
): Promise<string> {
const extraHeaders = getHeaders();
@@ -329,7 +536,11 @@ async function generateInlineScriptLock(
script_path: scriptPath,
},
],
raw_deps: rawDeps,
raw_workspace_dependencies:
rawWorkspaceDependencies &&
Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies
: null,
entrypoint: scriptPath,
}),
}
@@ -386,7 +597,7 @@ export interface InferredSchemaResult {
*/
export async function inferRunnableSchemaFromFile(
appFolder: string,
runnableFilePath: string,
runnableFilePath: string
): Promise<InferredSchemaResult | undefined> {
// Extract runnable ID from file path (e.g., "myRunnable.inline_script.ts" -> "myRunnable")
const fileName = path.basename(runnableFilePath);
@@ -406,7 +617,7 @@ export async function inferRunnableSchemaFromFile(
// Read the app file to get the language
const appFilePath = path.join(appFolder, "raw_app.yaml");
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
const appFile = (await yamlParseFile(appFilePath)) as RawAppFile;
if (!appFile.runnables?.[runnableId]) {
log.warn(colors.yellow(`Runnable ${runnableId} not found in raw_app.yaml`));
@@ -423,10 +634,12 @@ export async function inferRunnableSchemaFromFile(
const inlineScript = runnable.inlineScript;
const language = inlineScript.language as SupportedLanguage;
// Read the actual content from the file
const fullFilePath = path.join(appFolder, "runnables", runnableFilePath);
const fullFilePath = path.join(
appFolder,
APP_BACKEND_FOLDER,
runnableFilePath
);
let content: string;
try {
content = await Deno.readTextFile(fullFilePath);
@@ -461,3 +674,144 @@ export async function inferRunnableSchemaFromFile(
return undefined;
}
}
function getAppFolders(elems: Record<string, any>, extension: string) {
return Object.keys(elems)
.filter((p) => p.endsWith(SEP + extension))
.map((p) => p.substring(0, p.length - (SEP + extension).length));
}
export async function generateLocksCommand(
opts: GlobalOptions & {
yes?: boolean;
dryRun?: boolean;
defaultTs?: "bun" | "deno";
} & SyncOptions,
appPath: string | undefined
) {
const { generateAppLocksInternal } = await import("./app_metadata.ts");
const { elementsToMap, FSFSElement } = await import("../sync/sync.ts");
const { ignoreF } = await import("../sync/sync.ts");
const { Confirm } = await import("../../../deps.ts");
if (appPath == "") {
appPath = undefined;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
if (appPath) {
//TODO: Generate metadata for a specific raw app but handle normal apps to
throw new Error("Not implemented");
// Generate metadata for a specific app
// await generateAppLocksInternal(
// appPath,
// true,
// false,
// workspace,
// opts,
// false,
// false
// );
} else {
// Generate metadata for all apps
const ignore = await ignoreF(opts);
const elems = await elementsToMap(
await FSFSElement(Deno.cwd(), [], true),
(p, isD) => {
return (
ignore(p, isD) ||
(!isD &&
!p.endsWith(SEP + "raw_app.yaml") &&
!p.endsWith(SEP + "app.yaml"))
);
},
false,
{}
);
const rawAppFolders = getAppFolders(elems, "raw_app.yaml");
const appFolders = getAppFolders(elems, "app.yaml");
let hasAny = false;
log.info(
`Checking metadata for all apps (${appFolders.length}) and raw apps (${rawAppFolders.length})`
);
for (const appFolder of rawAppFolders) {
const candidate = await generateAppLocksInternal(
appFolder,
true,
true,
workspace,
opts,
false,
true
);
if (candidate) {
hasAny = true;
log.info(colors.green(`+ ${candidate}`));
}
}
for (const appFolder of appFolders) {
const candidate = await generateAppLocksInternal(
appFolder,
false,
true,
workspace,
opts,
false,
true
);
if (candidate) {
hasAny = true;
log.info(colors.green(`+ ${candidate}`));
}
}
if (hasAny) {
if (opts.dryRun) {
log.info(colors.gray(`Dry run complete.`));
return;
}
if (
!opts.yes &&
!(await Confirm.prompt({
message: "Update the metadata of the above apps?",
default: true,
}))
) {
return;
}
} else {
log.info(colors.green.bold("No metadata to update"));
return;
}
for (const appFolder of rawAppFolders) {
await generateAppLocksInternal(
appFolder,
true,
false,
workspace,
opts,
false,
true
);
}
for (const appFolder of appFolders) {
await generateAppLocksInternal(
appFolder,
false,
false,
workspace,
opts,
false,
true
);
}
}
}
cli/src/commands/app/apps.ts
+20 -12
View File
@@ -52,23 +52,31 @@ export function repopulateFields(runnables: Record<string, any>) {
}
});
}
export function replaceInlineScripts(rec: any, localPath: string) {
export function replaceInlineScripts(
rec: any,
localPath: string,
addType: boolean
) {
if (!rec) {
return;
}
if (typeof rec == "object") {
return Object.entries(rec).flatMap(([k, v]) => {
if (k == "runType") {
if (isVersionsGeq1585()) {
rec["type"] = "path";
} else {
rec["type"] = "runnableByPath";
if (addType) {
if (isVersionsGeq1585()) {
rec["type"] = "path";
} else {
rec["type"] = "runnableByPath";
}
}
} else if (k == "inlineScript" && typeof v == "object") {
if (isVersionsGeq1585()) {
rec["type"] = "inline";
} else {
rec["type"] = "runnableByName";
if (addType) {
if (isVersionsGeq1585()) {
rec["type"] = "inline";
} else {
rec["type"] = "runnableByName";
}
}
const o: Record<string, any> = v as any;
@@ -81,7 +89,7 @@ export function replaceInlineScripts(rec: any, localPath: string) {
o["lock"] = readInlinePathSync(basePath);
}
} else {
replaceInlineScripts(v, localPath);
replaceInlineScripts(v, localPath, addType);
}
});
}
@@ -125,7 +133,7 @@ export async function pushApp(
const path = localPath + "app.yaml";
const localApp = (await yamlParseFile(path)) as AppFile;
replaceInlineScripts(localApp.value, localPath);
replaceInlineScripts(localApp.value, localPath, true);
await generatingPolicy(
localApp,
remotePath,
@@ -234,7 +242,7 @@ const command = new Command()
"Default TypeScript runtime (bun or deno)"
)
.action(async (opts: any, appFolder: string | undefined) => {
const { generateLocksCommand } = await import("./raw_apps.ts");
const { generateLocksCommand } = await import("./app_metadata.ts");
await generateLocksCommand(opts, appFolder);
});
cli/src/commands/app/dev.ts
+56 -17
View File
@@ -7,6 +7,7 @@ import {
open,
windmillUtils,
yamlParseFile,
SEP,
} from "../../../deps.ts";
import { GlobalOptions } from "../../types.ts";
import * as http from "node:http";
@@ -16,7 +17,12 @@ import process from "node:process";
import { Buffer } from "node:buffer";
import { writeFileSync } from "node:fs";
import { WebSocketServer, WebSocket } from "npm:ws";
import { getDevBuildOptions, ensureNodeModules, createFrameworkPlugins, detectFrameworks } from "./bundle.ts";
import {
getDevBuildOptions,
ensureNodeModules,
createFrameworkPlugins,
detectFrameworks,
} from "./bundle.ts";
import { wmillTsDev as wmillTs } from "./wmillTsDev.ts";
import * as wmill from "../../../gen/services.gen.ts";
import { resolveWorkspace } from "../../core/context.ts";
@@ -24,7 +30,10 @@ import { requireLogin } from "../../core/auth.ts";
import { GLOBAL_CONFIG_OPT } from "../../core/conf.ts";
import { replaceInlineScripts } from "./apps.ts";
import { Runnable } from "./metadata.ts";
import { inferRunnableSchemaFromFile } from "./app_metadata.ts";
import {
APP_BACKEND_FOLDER,
inferRunnableSchemaFromFile,
} from "./app_metadata.ts";
const DEFAULT_PORT = 4000;
const DEFAULT_HOST = "localhost";
@@ -92,8 +101,8 @@ async function dev(opts: DevOptions) {
log.error(
colors.red(
`Error: The dev command must be run inside a .raw_app folder.\n` +
`Current directory: ${currentDirName}\n` +
`Please navigate to a folder ending with '.raw_app' before running this command.`
`Current directory: ${currentDirName}\n` +
`Please navigate to a folder ending with '.raw_app' before running this command.`
)
);
Deno.exit(1);
@@ -105,7 +114,7 @@ async function dev(opts: DevOptions) {
log.error(
colors.red(
`Error: raw_app.yaml not found in current directory.\n` +
`The dev command must be run in a .raw_app folder containing a raw_app.yaml file.`
`The dev command must be run in a .raw_app folder containing a raw_app.yaml file.`
)
);
Deno.exit(1);
@@ -133,7 +142,8 @@ async function dev(opts: DevOptions) {
// Detect frameworks to determine default entry point
const frameworks = detectFrameworks(process.cwd());
const defaultEntry = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx";
const defaultEntry =
frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx";
const entryPoint = opts.entry ?? defaultEntry;
// Verify entry point exists
@@ -206,7 +216,6 @@ async function dev(opts: DevOptions) {
},
};
// Create esbuild context
const ctx = await esbuild.context({
...buildOptions,
@@ -242,7 +251,7 @@ async function dev(opts: DevOptions) {
await ctx.rebuild();
// Watch runnables folder for changes
const runnablesPath = path.join(process.cwd(), "runnables");
const runnablesPath = path.join(process.cwd(), APP_BACKEND_FOLDER);
let runnablesWatcher: Deno.FsWatcher | undefined;
if (fs.existsSync(runnablesPath)) {
@@ -262,7 +271,10 @@ async function dev(opts: DevOptions) {
// Process each changed path with individual debouncing
for (const changedPath of event.paths) {
const relativePath = path.relative(process.cwd(), changedPath);
const relativeToRunnables = path.relative(runnablesPath, changedPath);
const relativeToRunnables = path.relative(
runnablesPath,
changedPath
);
// Skip non-modify events for schema inference
if (event.kind !== "modify" && event.kind !== "create") {
@@ -276,7 +288,9 @@ async function dev(opts: DevOptions) {
// Log the change event
log.info(
colors.cyan(`📝 Runnable changed [${event.kind}]: ${relativePath}`)
colors.cyan(
`📝 Runnable changed [${event.kind}]: ${relativePath}`
)
);
// Debounce schema inference per file (wait for typing to finish)
@@ -288,7 +302,9 @@ async function dev(opts: DevOptions) {
delete schemaInferenceTimeouts[changedPath];
try {
log.info(colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`));
log.info(
colors.cyan(`📝 Inferring schema for: ${relativeToRunnables}`)
);
// Infer schema for this runnable (returns schema in memory, doesn't write to file)
const result = await inferRunnableSchemaFromFile(
process.cwd(),
@@ -299,7 +315,15 @@ async function dev(opts: DevOptions) {
// log.info(colors.green(` Runnable ID: ${result.runnableId}`));
// Store inferred schema in memory
inferredSchemas[result.runnableId] = result.schema;
log.info(colors.green(` Inferred Schemas: ${JSON.stringify(inferredSchemas, null, 2)}`));
log.info(
colors.green(
` Inferred Schemas: ${JSON.stringify(
inferredSchemas,
null,
2
)}`
)
);
// Regenerate wmill.d.ts with updated schema from memory
await genRunnablesTs(inferredSchemas);
}
@@ -468,7 +492,9 @@ async function dev(opts: DevOptions) {
case "backendAsync": {
// Run a runnable asynchronously and return job ID immediately
log.info(
colors.blue(`[backendAsync] Running runnable async: ${runnable_id}`)
colors.blue(
`[backendAsync] Running runnable async: ${runnable_id}`
)
);
try {
const runnables = await loadRunnables();
@@ -628,7 +654,10 @@ const command = new Command()
.option("--host <host:string>", "Host to bind the dev server to", {
default: DEFAULT_HOST,
})
.option("--entry <entry:string>", "Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)")
.option(
"--entry <entry:string>",
"Entry point file (default: index.ts for Svelte/Vue, index.tsx otherwise)"
)
.option("--no-open", "Don't automatically open the browser")
.action(dev as any);
@@ -671,7 +700,11 @@ async function loadRunnables(): Promise<Record<string, Runnable>> {
const rawApp = (await yamlParseFile(
path.join(localPath, "raw_app.yaml")
)) as any;
replaceInlineScripts(rawApp.runnables, path.join(localPath, "runnables/"));
replaceInlineScripts(
rawApp.runnables,
path.join(localPath, APP_BACKEND_FOLDER) + SEP,
true
);
return rawApp?.runnables ?? {};
} catch (error: any) {
@@ -707,7 +740,10 @@ async function executeRunnable(
}
}
if ((runnable.type === "inline" || runnable.type === "runnableByName") && runnable.inlineScript) {
if (
(runnable.type === "inline" || runnable.type === "runnableByName") &&
runnable.inlineScript
) {
const inlineScript = runnable.inlineScript;
if (inlineScript.id !== undefined) {
requestBody.id = inlineScript.id;
@@ -719,7 +755,10 @@ async function executeRunnable(
lock: inlineScript.id === undefined ? inlineScript.lock : undefined,
cache_ttl: inlineScript.cache_ttl,
};
} else if ((runnable.type === "path" || runnable.type === "runnableByPath") && runnable.path) {
} else if (
(runnable.type === "path" || runnable.type === "runnableByPath") &&
runnable.path
) {
const runType = runnable.runType ?? "script";
requestBody.path =
runType !== "hubscript"
cli/src/commands/app/raw_apps.ts
+15 -101
View File
@@ -15,7 +15,7 @@ import { GlobalOptions, isSuperset } from "../../types.ts";
import { replaceInlineScripts, repopulateFields } from "./apps.ts";
import { createBundle, detectFrameworks } from "./bundle.ts";
import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts";
import { APP_BACKEND_FOLDER } from "./app_metadata.ts";
export interface AppFile {
runnables: any;
@@ -39,7 +39,12 @@ async function collectAppFiles(
if (entry.isDirectory) {
// Skip the runnables and node_modules subfolders
if (entry.name === "runnables" || entry.name === "node_modules" || entry.name === "dist" || entry.name === ".claude") {
if (
entry.name === APP_BACKEND_FOLDER ||
entry.name === "node_modules" ||
entry.name === "dist" ||
entry.name === ".claude"
) {
continue;
}
await readDirRecursive(fullPath + SEP, relativePath + SEP);
@@ -96,15 +101,20 @@ export async function pushRawApp(
}
const path = localPath + "raw_app.yaml";
const localApp = (await yamlParseFile(path)) as AppFile;
replaceInlineScripts(localApp.runnables, localPath + SEP + "runnables/");
repopulateFields(localApp.runnables)
replaceInlineScripts(
localApp.runnables,
localPath + SEP + APP_BACKEND_FOLDER + SEP,
true
);
repopulateFields(localApp.runnables);
await generatingPolicy(localApp, remotePath, localApp?.["public"] ?? false);
const files = await collectAppFiles(localPath);
async function createBundleRaw() {
log.info(colors.yellow.bold(`Creating raw app ${remotePath} bundle...`));
// Detect frameworks to determine entry point
const frameworks = detectFrameworks(localPath);
const entryFile = (frameworks.svelte || frameworks.vue) ? "index.ts" : "index.tsx";
const entryFile =
frameworks.svelte || frameworks.vue ? "index.ts" : "index.tsx";
const entryPoint = localPath + entryFile;
return await createBundle({
entryPoint: entryPoint,
@@ -183,102 +193,6 @@ export async function generatingPolicy(
}
}
export async function generateLocksCommand(
opts: GlobalOptions & {
yes?: boolean;
dryRun?: boolean;
defaultTs?: "bun" | "deno";
} & SyncOptions,
appPath: string | undefined
) {
const { generateAppLocksInternal } = await import("./app_metadata.ts");
const { elementsToMap, FSFSElement } = await import("../sync/sync.ts");
const { ignoreF } = await import("../sync/sync.ts");
const { Confirm } = await import("../../../deps.ts");
if (appPath == "") {
appPath = undefined;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
if (appPath) {
// Generate metadata for a specific app
await generateAppLocksInternal(
appPath,
false,
workspace,
opts,
false,
false,
);
} else {
// Generate metadata for all apps
const ignore = await ignoreF(opts);
const elems = await elementsToMap(
await FSFSElement(Deno.cwd(), [], true),
(p, isD) => {
return ignore(p, isD) || (!isD && !p.endsWith(SEP + "raw_app.yaml"));
},
false,
{}
);
const appFolders = Object.keys(elems)
.filter((p) => p.endsWith(SEP + "raw_app.yaml"))
.map((p) => p.substring(0, p.length - (SEP + "raw_app.yaml").length));
let hasAny = false;
log.info("Checking metadata for all apps:");
for (const appFolder of appFolders) {
const candidate = await generateAppLocksInternal(
appFolder,
true,
workspace,
opts,
false,
true,
);
if (candidate) {
hasAny = true;
log.info(colors.green(`+ ${candidate}`));
}
}
if (hasAny) {
if (opts.dryRun) {
log.info(colors.gray(`Dry run complete.`));
return;
}
if (
!opts.yes &&
!(await Confirm.prompt({
message: "Update the metadata of the above apps?",
default: true,
}))
) {
return;
}
} else {
log.info(colors.green.bold("No metadata to update"));
return;
}
for (const appFolder of appFolders) {
await generateAppLocksInternal(
appFolder,
false,
workspace,
opts,
false,
true,
);
}
}
}
async function pushRawAppCommand(
opts: GlobalOptions,
filePath: string,
cli/src/commands/flow/flow_metadata.ts
+21 -12
View File
@@ -18,9 +18,8 @@ import {
} from "../../utils/metadata.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
import { exts, } from "../script/script.ts";
import { exts } from "../script/script.ts";
import { FSFSElement } from "../sync/sync.ts";
import { Workspace } from "../workspace/workspace.ts";
import { FlowFile } from "./flow.ts";
@@ -67,8 +66,13 @@ export async function generateFlowLockInternal(
}
// Always get out-of-sync workspace dependencies
const rawWorkspaceDependencies: Record<string, string> = await getRawWorkspaceDependencies();
let hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
const rawWorkspaceDependencies: Record<string, string> =
await getRawWorkspaceDependencies();
let hashes = await generateFlowHash(
rawWorkspaceDependencies,
folder,
opts.defaultTs
);
const conf = await readLockfile();
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
@@ -115,7 +119,7 @@ export async function generateFlowLockInternal(
log,
folder + SEP!,
SEP,
changedScripts,
changedScripts
// (path: string, newPath: string) => Deno.renameSync(path, newPath),
// (path: string) => Deno.removeSync(path)
);
@@ -145,7 +149,11 @@ export async function generateFlowLockInternal(
);
}
hashes = await generateFlowHash(rawWorkspaceDependencies, folder, opts.defaultTs);
hashes = await generateFlowHash(
rawWorkspaceDependencies,
folder,
opts.defaultTs
);
await clearGlobalLock(folder);
for (const [path, hash] of Object.entries(hashes)) {
await updateMetadataGlobalLock(folder, hash, path);
@@ -153,8 +161,6 @@ export async function generateFlowLockInternal(
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
}
export async function updateFlow(
workspace: Workspace,
flow_value: FlowValue,
@@ -164,7 +170,9 @@ export async function updateFlow(
let rawResponse;
if (Object.keys(rawWorkspaceDependencies).length > 0) {
log.info(colors.blue("Using raw workspace dependencies for flow dependencies"));
log.info(
colors.blue("Using raw workspace dependencies for flow dependencies")
);
// generate the script lock running a dependency job in Windmill and update it inplace
const extraHeaders = getHeaders();
@@ -181,9 +189,10 @@ export async function updateFlow(
flow_value,
path: remotePath,
use_local_lockfiles: true,
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies
: null,
raw_workspace_dependencies:
Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies
: null,
}),
}
);
cli/src/commands/sync/sync.ts
+60 -8
View File
@@ -36,6 +36,7 @@ import {
import { handleFile } from "../script/script.ts";
import {
deepEqual,
fetchRemoteVersion,
isFileResource,
isRawAppFile,
isWorkspaceDependencies,
@@ -74,8 +75,10 @@ import {
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
import { isExecutionModeAnonymous } from "../app/apps.ts";
import { generateAppLocksInternal } from "../app/app_metadata.ts";
import { updateGlobalVersions } from "./global.ts";
import {
APP_BACKEND_FOLDER,
generateAppLocksInternal,
} from "../app/app_metadata.ts";
// Merge CLI options with effective settings, preserving CLI flags as overrides
function mergeCliWithEffectiveOptions<
@@ -549,7 +552,7 @@ function ZipFSElement(
for (const s of inlineScripts) {
yield {
isDirectory: false,
path: path.join(finalPath, "runnables", s.path),
path: path.join(finalPath, APP_BACKEND_FOLDER, s.path),
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
@@ -1737,6 +1740,19 @@ export async function pull(
await generateAppLocksInternal(
change,
false,
true,
workspace,
opts,
true,
true
);
}
for (const change of tracker.apps) {
log.info(`Updating lock metadata for app ${change}`);
await generateAppLocksInternal(
change,
false,
false,
workspace,
opts,
true,
@@ -1983,6 +1999,7 @@ export async function push(
const staleScripts: string[] = [];
const staleFlows: string[] = [];
const staleApps: string[] = [];
for (const change of tracker.scripts) {
const stale = await generateScriptMetadataInternal(
@@ -2036,16 +2053,51 @@ export async function push(
log.info("");
}
const version = await fetchVersion(workspace.remote);
if (version) {
updateGlobalVersions(version);
for (const change of tracker.apps) {
const stale = await generateAppLocksInternal(
change,
false,
true,
workspace,
opts,
true,
true
);
if (stale) {
staleApps.push(stale);
}
}
log.info(colors.gray("Remote version: " + version));
for (const change of tracker.rawApps) {
const stale = await generateAppLocksInternal(
change,
true,
true,
workspace,
opts,
true,
true
);
if (stale) {
staleApps.push(stale);
}
}
if (staleApps.length > 0) {
log.warn(
"Stale apps locks found, you may want to update them using 'wmill app generate-locks' before pushing:"
);
for (const stale of staleApps) {
log.warn(stale);
}
log.info("");
}
await fetchRemoteVersion(workspace);
log.info(
`remote (${workspace.name}) <- local: ${changes.length} changes to apply`
);
// Handle JSON output for dry-run
if (opts.dryRun && opts.jsonOutput) {
const result = {
cli/src/main.ts
+1 -1
View File
@@ -25,7 +25,6 @@ import instance from "./commands/instance/instance.ts";
import workerGroups from "./commands/worker-groups/worker_groups.ts";
import dev from "./commands/dev/dev.ts";
import { fetchVersion } from "./core/context.ts";
import { GlobalOptions } from "./types.ts";
import { OpenAPI } from "../gen/index.ts";
import { getHeaders, getIsWin } from "./utils/utils.ts";
@@ -39,6 +38,7 @@ import queues from "./commands/queues/queues.ts";
import dependencies from "./commands/dependencies/dependencies.ts";
import init from "./commands/init/init.ts";
import jobs from "./commands/jobs/jobs.ts";
import { fetchVersion } from "./core/context.ts";
export {
flow,
cli/src/utils/utils.ts
+14 -2
View File
@@ -4,6 +4,8 @@
import { colors, encodeHex, log, SEP } from "../../deps.ts";
import crypto from "node:crypto";
import { fetchVersion } from "../core/context.ts";
import { updateGlobalVersions } from "../commands/sync/global.ts";
export function deepEqual<T>(a: T, b: T): boolean {
if (a === b) return true;
@@ -151,11 +153,11 @@ export function isFileResource(path: string): boolean {
}
export function isRawAppFile(path: string): boolean {
return path.includes(".raw_app" + SEP) ;
return path.includes(".raw_app" + SEP);
}
export function isWorkspaceDependencies(path: string): boolean {
return path.startsWith("dependencies/")
return path.startsWith("dependencies/");
}
export function printSync(input: string | Uint8Array, to = Deno.stdout) {
@@ -263,3 +265,13 @@ export function writeIfChanged(path: string, content: string): boolean {
Deno.writeTextFileSync(path, content);
return true; // File was written
}
export async function fetchRemoteVersion(
workspace: Workspace
): Promise<string> {
const version = await fetchVersion(workspace.remote);
if (version) {
updateGlobalVersions(version);
}
log.info(colors.gray("Remote version: " + version));
}