Merge remote-tracking branch 'origin/main' into cli-onbehaflof

This commit is contained in:
wendrul
2026-03-26 10:57:37 +01:00
1007 changed files with 62352 additions and 12513 deletions
+1
View File
@@ -12,6 +12,7 @@ const parserPackages = [
"windmill-parser-wasm-yaml", "windmill-parser-wasm-csharp",
"windmill-parser-wasm-nu", "windmill-parser-wasm-java",
"windmill-parser-wasm-ruby",
"windmill-parser-wasm-py-imports",
];
const parserExternals = parserPackages.flatMap(p => ["--external", p]);
+5 -2
View File
@@ -24,10 +24,11 @@
"windmill-parser-wasm-nu": "*",
"windmill-parser-wasm-php": "*",
"windmill-parser-wasm-py": "*",
"windmill-parser-wasm-py-imports": "*",
"windmill-parser-wasm-regex": "*",
"windmill-parser-wasm-ruby": "*",
"windmill-parser-wasm-rust": "*",
"windmill-parser-wasm-ts": "*",
"windmill-parser-wasm-ts": "^1.659.1",
"windmill-parser-wasm-yaml": "*",
"windmill-yaml-validator": "1.1.1",
"ws": "8.18.0",
@@ -294,13 +295,15 @@
"windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.628.3", "", {}, "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="],
"windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.659.1", "", {}, "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="],
"windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.639.0", "", {}, "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ=="],
"windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="],
"windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.558.1", "", {}, "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="],
"windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.623.1", "", {}, "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw=="],
"windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.659.1", "", {}, "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="],
"windmill-parser-wasm-yaml": ["windmill-parser-wasm-yaml@1.593.0", "", {}, "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw=="],
+10 -4
View File
@@ -25,10 +25,11 @@
"windmill-parser-wasm-nu": "*",
"windmill-parser-wasm-php": "*",
"windmill-parser-wasm-py": "*",
"windmill-parser-wasm-py-imports": "*",
"windmill-parser-wasm-regex": "*",
"windmill-parser-wasm-ruby": "*",
"windmill-parser-wasm-rust": "*",
"windmill-parser-wasm-ts": "*",
"windmill-parser-wasm-ts": "^1.659.1",
"windmill-parser-wasm-yaml": "*",
"windmill-yaml-validator": "1.1.1",
"ws": "8.18.0",
@@ -1414,6 +1415,11 @@
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.628.3.tgz",
"integrity": "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="
},
"node_modules/windmill-parser-wasm-py-imports": {
"version": "1.659.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py-imports/-/windmill-parser-wasm-py-imports-1.659.1.tgz",
"integrity": "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="
},
"node_modules/windmill-parser-wasm-regex": {
"version": "1.639.0",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.639.0.tgz",
@@ -1430,9 +1436,9 @@
"integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ=="
},
"node_modules/windmill-parser-wasm-ts": {
"version": "1.647.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.647.1.tgz",
"integrity": "sha512-64iSAUMU5W/WtePqE1vtDvglDqtkiZVndyieYBVDX0nl7UuovS+wPgH/P3TEoKbR+FwAPacki0CX3DsEzZ/Yxw=="
"version": "1.659.1",
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.659.1.tgz",
"integrity": "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="
},
"node_modules/windmill-parser-wasm-yaml": {
"version": "1.593.0",
+2 -1
View File
@@ -32,10 +32,11 @@
"windmill-parser-wasm-nu": "*",
"windmill-parser-wasm-php": "*",
"windmill-parser-wasm-py": "*",
"windmill-parser-wasm-py-imports": "*",
"windmill-parser-wasm-regex": "*",
"windmill-parser-wasm-ruby": "*",
"windmill-parser-wasm-rust": "*",
"windmill-parser-wasm-ts": "*",
"windmill-parser-wasm-ts": "^1.659.1",
"windmill-parser-wasm-yaml": "*",
"windmill-yaml-validator": "1.1.1",
"ws": "8.18.0",
+3
View File
@@ -366,6 +366,9 @@ const command = new Command()
"Default TypeScript runtime (bun or deno)"
)
.action(async (opts: any, appFolder: string | undefined) => {
log.warn(
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
);
const { generateLocksCommand } = await import("./app_metadata.ts");
await generateLocksCommand(opts, appFolder);
});
+187 -79
View File
@@ -7,6 +7,7 @@ import { yamlParseFile } from "../../utils/yaml.ts";
import { stringify as yamlStringify } from "yaml";
import { GlobalOptions } from "../../types.ts";
import {
readLockfile,
checkifMetadataUptodate,
blueColor,
clearGlobalLock,
@@ -41,6 +42,8 @@ import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
import { getNonDottedPaths } from "../../utils/resource_folders.ts";
import { extractRelativeImports } from "../../utils/relative_imports.ts";
import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts";
const TOP_HASH = "__app_hash";
export const APP_BACKEND_FOLDER = "backend";
@@ -93,7 +96,16 @@ async function generateAppHash(
}
/**
* Updates locks for inline scripts in an app
* Result of generating app locks, including which scripts were updated
*/
export interface AppLocksResult {
path: string;
updatedScripts: string[];
}
/**
* Updates locks for inline scripts in an app.
* Returns the path if dry-run, or AppLocksResult with updated scripts if actual update occurred.
*/
export async function generateAppLocksInternal(
appFolder: string,
@@ -104,8 +116,10 @@ export async function generateAppLocksInternal(
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean
): Promise<string | void> {
noStaleMessage?: boolean,
legacyBehaviour?: boolean,
tree?: DoubleLinkedDependencyTree
): Promise<string | AppLocksResult | void> {
if (appFolder.endsWith(SEP)) {
appFolder = appFolder.substring(0, appFolder.length - 1);
}
@@ -116,9 +130,6 @@ export async function generateAppLocksInternal(
log.info(`Generating locks for app ${appFolder} at ${remote_path}`);
}
const rawWorkspaceDependencies: Record<string, string> =
await getRawWorkspaceDependencies();
// Read the app file first to filter workspace dependencies
const appFilePath = path.join(
appFolder,
@@ -126,38 +137,83 @@ export async function generateAppLocksInternal(
);
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
// Filter workspace dependencies based on inline scripts' languages and annotations
const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value;
const filteredDeps = await filterWorkspaceDependenciesForApp(
appValue,
rawWorkspaceDependencies,
appFolder
);
const folderNormalized = appFolder.replaceAll(SEP, "/");
let hashes = await generateAppHash(
filteredDeps,
appFolder,
rawApp,
opts.defaultTs
);
let filteredDeps: Record<string, string> = {};
const conf = await readLockfile();
const conf = await import("../../utils/metadata.ts").then((m) =>
m.readLockfile()
);
if (
await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)
) {
if (!noStaleMessage) {
log.info(
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
);
// New behaviour: tree-based dependency tracking
if (!legacyBehaviour && tree) {
if (dryRun) {
const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs);
const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH));
// For raw apps in new format, runnables are in separate files under backend/
let treeAppValue = structuredClone(appValue);
if (rawApp) {
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER);
const runnablesFromFiles = await loadRunnablesFromBackend(runnablesPath);
if (Object.keys(runnablesFromFiles).length > 0) {
treeAppValue = runnablesFromFiles;
}
}
// First pass: add inline scripts as separate nodes, then add app node importing them
const inlineScriptPaths: string[] = [];
await traverseAndProcessInlineScripts(treeAppValue, async (inlineScript, context) => {
if (!inlineScript.content || !inlineScript.language) {
return inlineScript;
}
let content = inlineScript.content;
// Resolve !inline references
if (typeof content === "string" && content.startsWith("!inline ")) {
const filePath = appFolder + SEP + content.replace("!inline ", "");
try {
content = await readFile(filePath, "utf-8");
} catch {
return inlineScript;
}
}
const treePath = folderNormalized + "/" + context.path.join("/");
const language = inlineScript.language as ScriptLanguage;
const imports = await extractRelativeImports(content, treePath, language);
await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, appFolder, false);
inlineScriptPaths.push(treePath);
return inlineScript;
});
await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "app", folderNormalized, appFolder, isDirectlyStale, rawApp);
return;
}
// Second pass: get mismatched workspace deps from tree
// TODO: pass raw workspace deps more precisely to every inline script lock generation call
// (currently we pass the union of all mismatched deps filtered for the whole app)
filteredDeps = await filterWorkspaceDependenciesForApp(appValue, tree.getMismatchedWorkspaceDeps(), appFolder);
} else {
// Legacy behaviour
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder);
const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs);
const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH));
if (!isDirectlyStale) {
if (!noStaleMessage) {
log.info(
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
return remote_path;
}
return;
} else if (dryRun) {
return remote_path;
}
if (Object.keys(filteredDeps).length > 0) {
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
log.info(
(await blueColor())(
`Found workspace dependencies (${workspaceDependenciesLanguages
@@ -167,7 +223,11 @@ export async function generateAppLocksInternal(
);
}
let updatedScripts: string[] = [];
if (!justUpdateMetadataLock) {
const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs);
const changedScripts = [];
// Find hashes that do not correspond to previous hashes
for (const [scriptPath, hash] of Object.entries(hashes)) {
@@ -179,10 +239,18 @@ export async function generateAppLocksInternal(
}
}
if (changedScripts.length > 0) {
log.info(
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
);
// Get temp_script_refs from tree for relative import resolution
const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized);
// In tree mode, the tree already verified this app is stale (possibly via dependency change).
// Per-script hashes only detect content changes, not transitive dependency changes,
// so we must regenerate locks for all inline scripts regardless.
if (changedScripts.length > 0 || (tree && !legacyBehaviour)) {
if (!noStaleMessage) {
log.info(
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
);
}
if (rawApp) {
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER);
@@ -199,13 +267,15 @@ export async function generateAppLocksInternal(
replaceInlineScripts(runnables, runnablesPath + SEP, false);
// Update the app runnables with new locks (writes to separate files)
await updateRawAppRunnables(
updatedScripts = await updateRawAppRunnables(
workspace,
runnables,
remote_path,
appFolder,
filteredDeps,
opts.defaultTs
opts.defaultTs,
noStaleMessage,
tempScriptRefs
);
// Note: updateRawAppRunnables now writes each runnable to its own file
} else {
@@ -215,14 +285,18 @@ export async function generateAppLocksInternal(
replaceInlineScripts(normalAppFile.value, appFolder + SEP, false);
// Update the app value with new locks
normalAppFile.value = await updateAppInlineScripts(
const result = await updateAppInlineScripts(
workspace,
normalAppFile.value,
remote_path,
appFolder,
filteredDeps,
opts.defaultTs
opts.defaultTs,
noStaleMessage,
tempScriptRefs
);
normalAppFile.value = result.value;
updatedScripts = result.updatedScripts;
// Write the updated app file (only for normal apps, raw apps use separate files)
writeIfChanged(
@@ -230,23 +304,28 @@ export async function generateAppLocksInternal(
yamlStringify(appFile as Record<string, any>, yamlOptions)
);
}
} else {
} else if (!noStaleMessage) {
log.info(colors.gray(`No scripts changed in ${appFolder}`));
}
}
// Regenerate hashes after updates
hashes = await generateAppHash(
filteredDeps,
// Non-legacy mode excludes workspace deps from hash (tracked via tree instead)
const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps;
const finalHashes = await generateAppHash(
depsForHash,
appFolder,
rawApp,
opts.defaultTs
);
await clearGlobalLock(appFolder);
for (const [scriptPath, hash] of Object.entries(hashes)) {
for (const [scriptPath, hash] of Object.entries(finalHashes)) {
await updateMetadataGlobalLock(appFolder, hash, scriptPath);
}
log.info(colors.green(`App ${remote_path} lockfiles updated`));
if (!noStaleMessage) {
log.info(colors.green(`App ${remote_path} lockfiles updated`));
}
return { path: remote_path, updatedScripts };
}
/**
@@ -336,6 +415,7 @@ async function traverseAndProcessInlineScripts(
* Updates locks for all runnables in a raw app, generating locks inline script by inline script.
* Writes each runnable to its own YAML file in the backend folder (new format).
* Also writes content and lock files to the runnables folder.
* Returns the list of runnable IDs that had their locks updated.
*/
async function updateRawAppRunnables(
workspace: Workspace,
@@ -343,8 +423,11 @@ async function updateRawAppRunnables(
remotePath: string,
appFolder: string,
rawDeps?: Record<string, string>,
defaultTs: "bun" | "deno" = "bun"
): Promise<void> {
defaultTs: "bun" | "deno" = "bun",
noStaleMessage?: boolean,
tempScriptRefs?: Record<string, string>
): Promise<string[]> {
const updatedRunnables: string[] = [];
const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER);
// Ensure runnables folder exists
@@ -410,12 +493,11 @@ async function updateRawAppRunnables(
continue;
}
log.info(
colors.gray(
`Generating lock for runnable ${runnableId} (${language})
}`
)
);
if (!noStaleMessage) {
log.info(
colors.gray(`Generating lock for runnable ${runnableId} (${language})`)
);
}
try {
const lock = await generateInlineScriptLock(
@@ -423,7 +505,8 @@ async function updateRawAppRunnables(
content,
language,
`${remotePath}/${runnableId}`,
rawDeps
rawDeps,
tempScriptRefs
);
// Determine file extension for this language
@@ -455,11 +538,15 @@ async function updateRawAppRunnables(
// Write the runnable to its own YAML file
writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable);
log.info(
colors.gray(
` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
)
);
updatedRunnables.push(runnableId);
if (!noStaleMessage) {
log.info(
colors.gray(
` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
)
);
}
} catch (error: any) {
log.error(
colors.red(
@@ -470,11 +557,14 @@ async function updateRawAppRunnables(
writeRunnableToBackend(runnablesFolder, runnableId, runnable);
}
}
return updatedRunnables;
}
/**
* Updates locks for all inline scripts in a normal app, similar to updateRawAppRunnables
* but for the app.value structure instead of app.runnables
* but for the app.value structure instead of app.runnables.
* Returns a tuple of [updated app value, list of script names that were updated].
*/
async function updateAppInlineScripts(
workspace: Workspace,
@@ -482,9 +572,12 @@ async function updateAppInlineScripts(
remotePath: string,
appFolder: string,
rawDeps?: Record<string, string>,
defaultTs: "bun" | "deno" = "bun"
): Promise<any> {
defaultTs: "bun" | "deno" = "bun",
noStaleMessage?: boolean,
tempScriptRefs?: Record<string, string>
): Promise<{ value: any; updatedScripts: string[] }> {
const pathAssigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() });
const updatedScripts: string[] = [];
const processor: InlineScriptProcessor = async (inlineScript, context) => {
const language = inlineScript.language as SupportedLanguage;
@@ -514,20 +607,23 @@ async function updateAppInlineScripts(
try {
let lock: string | undefined;
if (language !== "frontend") {
log.info(
colors.gray(
`Generating lock for inline script "${scriptName}" at ${context.path.join(
"."
)} (${language})`
)
);
if (!noStaleMessage) {
log.info(
colors.gray(
`Generating lock for inline script "${scriptName}" at ${context.path.join(
"."
)} (${language})`
)
);
}
lock = await generateInlineScriptLock(
workspace,
content,
language,
scriptPath,
rawDeps
rawDeps,
tempScriptRefs
);
}
// Determine file extension for this language (following extractInlineScriptsForApps pattern)
@@ -549,11 +645,18 @@ async function updateAppInlineScripts(
const inlineLockRef =
lock && lock !== "" ? `!inline ${basePath}lock` : "";
log.info(
colors.gray(
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
)
);
if (!noStaleMessage) {
log.info(
colors.gray(
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
)
);
}
// Track that this script was updated (only for non-frontend scripts that needed locks)
if (language !== "frontend") {
updatedScripts.push(scriptName);
}
return {
...inlineScript,
@@ -573,7 +676,8 @@ async function updateAppInlineScripts(
}
};
return await traverseAndProcessInlineScripts(appValue, processor);
const updatedValue = await traverseAndProcessInlineScripts(appValue, processor);
return { value: updatedValue, updatedScripts };
}
/**
@@ -584,7 +688,8 @@ async function generateInlineScriptLock(
content: string,
language: string,
scriptPath: string,
rawWorkspaceDependencies: Record<string, string> | undefined
rawWorkspaceDependencies: Record<string, string> | undefined,
tempScriptRefs?: Record<string, string>
): Promise<string> {
// Filter workspace dependencies to only include those matching this script's language and annotations
const filteredDeps = rawWorkspaceDependencies
@@ -615,6 +720,9 @@ async function generateInlineScriptLock(
? filteredDeps
: null,
entrypoint: scriptPath,
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
? { temp_script_refs: tempScriptRefs }
: {}),
}),
}
);
@@ -767,7 +875,7 @@ export async function inferRunnableSchemaFromFile(
}
}
function getAppFolders(elems: Record<string, any>, extension: string) {
export function getAppFolders(elems: Record<string, any>, extension: string) {
return Object.keys(elems)
.filter((p) => p.endsWith(SEP + extension))
.map((p) => p.substring(0, p.length - (SEP + extension).length));
+18 -6
View File
@@ -16,33 +16,34 @@ import { resolveWorkspace } from "../../core/context.ts";
import {
SyncOptions,
mergeConfigWithConfigFile,
readConfigFile,
} from "../../core/conf.ts";
import { exts, removeExtensionToPath } from "../script/script.ts";
import { inferContentTypeFromFilePath } from "../../utils/script_common.ts";
import { OpenFlow } from "../../../gen/types.gen.ts";
import { FlowFile } from "../flow/flow.ts";
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { replaceInlineScripts, replaceAllPathScriptsWithLocal } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { parseMetadataFile } from "../../utils/metadata.ts";
import {
getFolderSuffixWithSep,
getMetadataFileName,
extractFolderPath,
} from "../../utils/resource_folders.ts";
import { listSyncCodebases } from "../../utils/codebase.ts";
import { createPreviewLocalScriptReader } from "../../utils/local_path_scripts.ts";
const PORT = 3001;
async function dev(opts: GlobalOptions & SyncOptions) {
opts = await mergeConfigWithConfigFile(opts);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
log.info("Started dev mode");
const conf = await readConfigFile();
let currentLastEdit: LastEditScript | LastEditFlow | undefined = undefined;
const fsWatcher = watch(".", { recursive: true });
const base = await realpath(".");
opts = await mergeConfigWithConfigFile(opts);
const ignore = await ignoreF(opts);
const codebases = await listSyncCodebases(opts);
const changesTimeouts: Record<string, ReturnType<typeof setTimeout>> = {};
function watchChanges() {
@@ -56,7 +57,11 @@ async function dev(opts: GlobalOptions & SyncOptions) {
}
changesTimeouts[key] = setTimeout(async () => {
delete changesTimeouts[key];
await loadPaths([filePath]);
await loadPaths([filePath]).catch((error) => {
log.error(
`Failed to reload ${filePath}: ${error instanceof Error ? error.message : error}`
);
});
}, 100);
});
fsWatcher.on("error", (err) => {
@@ -94,6 +99,13 @@ async function dev(opts: GlobalOptions & SyncOptions) {
SEP,
undefined,
);
// Replace PathScript modules with local file content so dev mode uses local versions
const localScriptReader = createPreviewLocalScriptReader({
exts,
defaultTs: opts.defaultTs,
codebases,
});
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
currentLastEdit = {
type: "flow",
flow: localFlow,
@@ -105,7 +117,7 @@ async function dev(opts: GlobalOptions & SyncOptions) {
const content = await readFile(cpath, "utf-8");
const splitted = cpath.split(".");
const wmPath = splitted[0];
const lang = inferContentTypeFromFilePath(cpath, conf.defaultTs);
const lang = inferContentTypeFromFilePath(cpath, opts.defaultTs);
const typed =
(await parseMetadataFile(
removeExtensionToPath(cpath),
+159 -6
View File
@@ -18,10 +18,21 @@ import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts";
import { Flow, OpenFlowWPath } from "../../../gen/types.gen.ts";
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import {
collectPathScriptPaths,
replaceInlineScripts,
replaceAllPathScriptsWithLocal,
} from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { generateFlowLockInternal } from "./flow_metadata.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { resolvePermissionedAsRule } from "../../core/permissioned_as.ts";
import { exts } from "../script/script.ts";
import type { SyncCodebase } from "../../utils/codebase.ts";
import { listSyncCodebases } from "../../utils/codebase.ts";
import {
createPreviewLocalScriptReader,
resolvePreviewLocalScriptState,
} from "../../utils/local_path_scripts.ts";
export interface FlowFile {
summary: string;
@@ -31,6 +42,90 @@ export interface FlowFile {
on_behalf_of_email?: string;
}
function normalizeOptionalString(value: string | null | undefined): string | undefined {
return typeof value === "string" && value.trim() === "" ? undefined : value ?? undefined;
}
function normalizeComparableContent(value: string | undefined): string | undefined {
return value?.replaceAll("\r\n", "\n").replace(/\n$/, "");
}
async function findDivergedLocalPathScripts(
workspaceId: string,
scriptPaths: string[],
opts: {
exts: string[];
defaultTs?: "bun" | "deno";
codebases: SyncCodebase[];
}
): Promise<{ changed: string[]; missing: string[] }> {
const changed: string[] = [];
const missing: string[] = [];
for (const scriptPath of scriptPaths) {
const localScript = await resolvePreviewLocalScriptState(scriptPath, opts);
if (!localScript) {
continue;
}
let remoteScript;
try {
remoteScript = await wmill.getScriptByPath({
workspace: workspaceId,
path: scriptPath,
});
} catch {
missing.push(scriptPath);
continue;
}
const remoteLock = normalizeOptionalString(remoteScript.lock);
const diverged =
normalizeComparableContent(localScript.content) !==
normalizeComparableContent(remoteScript.content) ||
localScript.language !== remoteScript.language ||
(localScript.lock !== undefined &&
normalizeComparableContent(localScript.lock) !==
normalizeComparableContent(remoteLock)) ||
localScript.tag !== normalizeOptionalString(remoteScript.tag) ||
localScript.codebaseDigest !== normalizeOptionalString(remoteScript.codebase);
if (diverged) {
changed.push(scriptPath);
}
}
return { changed, missing };
}
function warnAboutLocalPathScriptDivergence(
divergence: { changed: string[]; missing: string[] }
): void {
if (divergence.changed.length === 0 && divergence.missing.length === 0) {
return;
}
const details: string[] = [];
if (divergence.changed.length > 0) {
details.push(
`These workspace scripts differ from the deployed version:\n${divergence.changed
.map((path) => `- ${path}`)
.join("\n")}`
);
}
if (divergence.missing.length > 0) {
details.push(
`These scripts do not exist in the workspace yet:\n${divergence.missing
.map((path) => `- ${path}`)
.join("\n")}`
);
}
log.warn(
`Using local PathScript files for flow preview.\n${details.join("\n")}\nUse --remote to preview deployed workspace scripts instead.`
);
}
const alreadySynced: string[] = [];
export async function pushFlow(
@@ -60,13 +155,20 @@ export async function pushFlow(
}
const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile;
const fileReader = async (path: string) => await readFile(localPath + path, "utf-8");
await replaceInlineScripts(
localFlow.value.modules,
async (path: string) => await readFile(localPath + path, "utf-8"),
fileReader,
log,
localPath,
SEP
);
if (localFlow.value.failure_module) {
await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, localPath, SEP);
}
if (localFlow.value.preprocessor_module) {
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, localPath, SEP);
}
// Build preserve flags for permissioned_as
const preserveFields: Partial<OpenFlowWPath> = {};
@@ -256,11 +358,17 @@ async function preview(
opts: GlobalOptions & {
data?: string;
silent: boolean;
},
remote?: boolean;
} & SyncOptions,
flowPath: string
) {
const useLocalPathScripts = !opts.remote;
if (useLocalPathScripts) {
opts = await mergeConfigWithConfigFile(opts);
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
const codebases = useLocalPathScripts ? listSyncCodebases(opts) : [];
// Normalize path - ensure it's a directory path to a .flow folder
if (!flowPath.endsWith(".flow") && !flowPath.endsWith(".flow" + SEP)) {
@@ -282,13 +390,45 @@ async function preview(
const localFlow = (await yamlParseFile(flowPath + "flow.yaml")) as FlowFile;
// Replace inline scripts with their actual content
const fileReader = async (path: string) => await readFile(flowPath + path, "utf-8");
await replaceInlineScripts(
localFlow.value.modules,
async (path: string) => await readFile(flowPath + path, "utf-8"),
fileReader,
log,
flowPath,
SEP
);
if (localFlow.value.failure_module) {
await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, flowPath, SEP);
}
if (localFlow.value.preprocessor_module) {
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, flowPath, SEP);
}
if (useLocalPathScripts) {
const scriptPaths = collectPathScriptPaths(localFlow.value);
if (scriptPaths.length > 0) {
const divergence = await findDivergedLocalPathScripts(
workspace.workspaceId,
scriptPaths,
{
exts,
defaultTs: opts.defaultTs,
codebases,
}
);
if (!opts.silent) {
warnAboutLocalPathScriptDivergence(divergence);
}
}
const localScriptReader = createPreviewLocalScriptReader({
exts,
defaultTs: opts.defaultTs,
codebases,
});
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
}
const input = opts.data ? await resolve(opts.data) : {};
@@ -324,12 +464,16 @@ async function preview(
}
}
async function generateLocks(
export async function generateLocks(
opts: GlobalOptions & {
yes?: boolean;
dryRun?: boolean;
} & SyncOptions,
folder: string | undefined
) {
log.warn(
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
);
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
@@ -374,6 +518,10 @@ async function generateLocks(
}
if (hasAny) {
if (opts.dryRun) {
log.info(colors.gray("Dry run complete."));
return;
}
if (
!opts.yes &&
!(await Confirm.prompt({
@@ -481,7 +629,7 @@ const command = new Command()
.action(run as any)
.command(
"preview",
"preview a local flow without deploying it. Runs the flow definition from local files."
"preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default."
)
.arguments("<flow_path:string>")
.option(
@@ -492,6 +640,10 @@ const command = new Command()
"-s --silent",
"Do not output anything other then the final output. Useful for scripting."
)
.option(
"--remote",
"Use deployed workspace scripts for PathScript steps instead of local files."
)
.action(preview as any)
.command(
"generate-locks",
@@ -499,6 +651,7 @@ const command = new Command()
)
.arguments("[flow:file]")
.option("--yes", "Skip confirmation prompt")
.option("--dry-run", "Perform a dry run without making changes")
.option(
"-i --includes <patterns:file[]>",
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)"
+165 -35
View File
@@ -19,6 +19,7 @@ import {
} from "../../utils/metadata.ts";
import { ScriptLanguage } from "../../utils/script_common.ts";
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { newPathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
import { exts } from "../script/script.ts";
@@ -28,7 +29,9 @@ import { FlowFile } from "./flow.ts";
import { FlowValue } from "../../../gen/types.gen.ts";
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { workspaceDependenciesLanguages } from "../../utils/script_common.ts";
import { extractNameFromFolder, getFolderSuffix } from "../../utils/resource_folders.ts";
import { extractNameFromFolder, getFolderSuffix, getNonDottedPaths } from "../../utils/resource_folders.ts";
import { extractRelativeImports } from "../../utils/relative_imports.ts";
import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts";
const TOP_HASH = "__flow_hash";
async function generateFlowHash(
@@ -50,6 +53,14 @@ async function generateFlowHash(
}
return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
}
/**
* Result of generating flow locks, including which scripts were updated
*/
export interface FlowLocksResult {
path: string;
updatedScripts: string[];
}
export async function generateFlowLockInternal(
folder: string,
dryRun: boolean,
@@ -58,8 +69,10 @@ export async function generateFlowLockInternal(
defaultTs?: "bun" | "deno";
},
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean
): Promise<string | void> {
noStaleMessage?: boolean,
legacyBehaviour?: boolean,
tree?: DoubleLinkedDependencyTree
): Promise<string | FlowLocksResult | void> {
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
}
@@ -68,36 +81,70 @@ export async function generateFlowLockInternal(
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
}
// Always get out-of-sync workspace dependencies
const rawWorkspaceDependencies: Record<string, string> =
await getRawWorkspaceDependencies();
const flowValue = (await yamlParseFile(
folder! + SEP + "flow.yaml"
)) as FlowFile;
// Filter workspace dependencies based on inline scripts' languages and annotations
const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
let hashes = await generateFlowHash(
filteredDeps,
folder,
const folderNormalized = folder.replaceAll(SEP, "/");
const inlineScriptsForTree = extractInlineScriptsForFlows(
structuredClone(flowValue.value.modules),
{},
SEP,
opts.defaultTs
);
).filter(s => !s.is_lock);
let filteredDeps: Record<string, string> = {};
const conf = await readLockfile();
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
if (!noStaleMessage) {
log.info(
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
);
if (!legacyBehaviour && tree) {
if (dryRun) {
const inlineScriptPaths: string[] = [];
for (const script of inlineScriptsForTree) {
let content = script.content;
if (content.startsWith("!inline ")) {
const filePath = folder + SEP + content.replace("!inline ", "");
try {
content = await readFile(filePath, "utf-8");
} catch {
continue;
}
}
const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path));
const language = script.language as ScriptLanguage;
const imports = await extractRelativeImports(content, treePath, language);
await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, folder, false);
inlineScriptPaths.push(treePath);
}
const hashes = await generateFlowHash({}, folder, opts.defaultTs);
const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH));
await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale);
return;
}
// Second pass: get mismatched workspace deps from tree
filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, tree.getMismatchedWorkspaceDeps(), folder);
} else {
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs);
const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH));
if (!isDirectlyStale) {
if (!noStaleMessage) {
log.info(
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
return remote_path;
}
return;
} else if (dryRun) {
return remote_path;
}
if (Object.keys(filteredDeps).length > 0) {
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
log.info(
(await blueColor())(
`Found workspace dependencies (${workspaceDependenciesLanguages
@@ -108,8 +155,25 @@ export async function generateFlowLockInternal(
}
let changedScripts: string[] = [];
// Build mapping from on-disk file names (hash keys like "a.py") to tree paths
// (like "folder/a.inline_script"). The tree uses extractInlineScriptsForFlows without
// a path assigner, so paths always have .inline_script suffix, but on-disk files
// may not (non-dotted mode).
const fileToTreePath = new Map<string, string>();
for (const script of inlineScriptsForTree) {
const c = script.content;
if (c.startsWith("!inline ")) {
const fileName = c.replace("!inline ", "");
const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path));
fileToTreePath.set(fileName, treePath);
}
}
if (!justUpdateMetadataLock) {
const changedScripts = [];
const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs);
//find hashes that do not correspond to previous hashes
for (const [path, hash] of Object.entries(hashes)) {
if (path == TOP_HASH) {
@@ -120,30 +184,61 @@ export async function generateFlowLockInternal(
}
}
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
if (!noStaleMessage) {
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
}
const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8");
// In tree mode, use the tree's staleness info (which includes transitive dependency changes)
// to determine which scripts need relocking, instead of only content-changed ones.
const locksToRemove = (tree && !legacyBehaviour)
? Object.keys(hashes).filter(k => {
if (k === TOP_HASH) return false;
const treePath = fileToTreePath.get(k)
?? (folderNormalized + "/" + path.basename(k, path.extname(k)));
return tree.isStale(treePath);
})
: changedScripts;
await replaceInlineScripts(
flowValue.value.modules,
async (path: string) => await readFile(folder + SEP + path, "utf-8"),
fileReader,
log,
folder + SEP!,
SEP,
changedScripts
locksToRemove
);
if (flowValue.value.failure_module) {
await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, locksToRemove);
}
if (flowValue.value.preprocessor_module) {
await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, locksToRemove);
}
//removeChangedLocks
const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized);
flowValue.value = await updateFlow(
workspace,
flowValue.value,
remote_path,
filteredDeps
filteredDeps,
tempScriptRefs
);
const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", {
skipInlineScriptSuffix: getNonDottedPaths(),
});
const inlineScripts = extractInlineScriptsForFlows(
flowValue.value.modules,
{},
SEP,
opts.defaultTs
opts.defaultTs,
lockAssigner
);
if (flowValue.value.failure_module) {
inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.failure_module], {}, SEP, opts.defaultTs, lockAssigner));
}
if (flowValue.value.preprocessor_module) {
inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.preprocessor_module], {}, SEP, opts.defaultTs, lockAssigner));
}
inlineScripts.forEach((s) => {
writeIfChanged(process.cwd() + SEP + folder + SEP + s.path, s.content);
});
@@ -155,16 +250,36 @@ export async function generateFlowLockInternal(
);
}
hashes = await generateFlowHash(
filteredDeps,
// Non-legacy mode excludes workspace deps from hash (tracked via tree instead)
const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps;
const finalHashes = await generateFlowHash(
depsForHash,
folder,
opts.defaultTs
);
await clearGlobalLock(folder);
for (const [path, hash] of Object.entries(hashes)) {
for (const [path, hash] of Object.entries(finalHashes)) {
await updateMetadataGlobalLock(folder, hash, path);
}
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
if (!noStaleMessage) {
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
}
// Return the list of updated scripts (extract just the filename from the path)
// In tree mode, use the same staleness-aware list we used for lock removal
const relocked = (tree && !legacyBehaviour)
? Object.keys(finalHashes).filter(k => {
if (k === TOP_HASH) return false;
const treePath = fileToTreePath.get(k)
?? (folderNormalized + "/" + path.basename(k, path.extname(k)));
return tree.isStale(treePath);
})
: changedScripts;
const updatedScripts = relocked.map(p => {
const parts = p.split(SEP);
return parts[parts.length - 1].replace(/\.[^.]+$/, ""); // Remove extension
});
return { path: remote_path, updatedScripts };
}
/**
@@ -176,7 +291,15 @@ async function filterWorkspaceDependenciesForFlow(
rawWorkspaceDependencies: Record<string, string>,
folder: string
): Promise<Record<string, string>> {
const inlineScripts = extractInlineScriptsForFlows(structuredClone(flowValue.modules), {}, SEP, undefined);
const clonedValue = structuredClone(flowValue);
const depAssigner = newPathAssigner("bun");
const inlineScripts = extractInlineScriptsForFlows(clonedValue.modules, {}, SEP, undefined, depAssigner);
if (clonedValue.failure_module) {
inlineScripts.push(...extractInlineScriptsForFlows([clonedValue.failure_module], {}, SEP, undefined, depAssigner));
}
if (clonedValue.preprocessor_module) {
inlineScripts.push(...extractInlineScriptsForFlows([clonedValue.preprocessor_module], {}, SEP, undefined, depAssigner));
}
// Filter out lock files and map to common interface
const scripts = inlineScripts
@@ -190,7 +313,8 @@ export async function updateFlow(
workspace: Workspace,
flow_value: FlowValue,
remotePath: string,
rawWorkspaceDependencies: Record<string, string>
rawWorkspaceDependencies: Record<string, string>,
tempScriptRefs?: Record<string, string>
): Promise<FlowValue | undefined> {
let rawResponse;
@@ -215,6 +339,9 @@ export async function updateFlow(
path: remotePath,
use_local_lockfiles: true,
raw_workspace_dependencies: rawWorkspaceDependencies,
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
? { temp_script_refs: tempScriptRefs }
: {}),
}),
}
);
@@ -233,6 +360,9 @@ export async function updateFlow(
body: JSON.stringify({
flow_value,
path: remotePath,
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
? { temp_script_refs: tempScriptRefs }
: {}),
}),
}
);
@@ -0,0 +1,446 @@
import { Command } from "@cliffy/command";
import { Confirm } from "@cliffy/prompt/confirm";
import { colors } from "@cliffy/ansi/colors";
import { sep as SEP } from "node:path";
import { GlobalOptions } from "../../types.ts";
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
import { resolveWorkspace } from "../../core/context.ts";
import { requireLogin } from "../../core/auth.ts";
import * as log from "../../core/log.ts";
import {
generateScriptMetadataInternal,
getRawWorkspaceDependencies,
readLockfile,
checkifMetadataUptodate,
} from "../../utils/metadata.ts";
import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts";
import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts";
import {
elementsToMap,
FSFSElement,
ignoreF,
} from "../sync/sync.ts";
import { exts } from "../script/script.ts";
import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts";
import { listSyncCodebases } from "../../utils/codebase.ts";
import {
DoubleLinkedDependencyTree,
uploadScripts,
ItemType,
} from "../../utils/dependency_tree.ts";
interface StaleItem {
type: ItemType;
path: string;
folder: string;
isRawApp?: boolean;
staleReason?: string;
}
async function generateMetadata(
opts: GlobalOptions & {
yes?: boolean;
lockOnly?: boolean;
schemaOnly?: boolean;
dryRun?: boolean;
skipScripts?: boolean;
skipFlows?: boolean;
skipApps?: boolean;
strictFolderBoundaries?: boolean;
} & SyncOptions,
folder?: string
) {
if (folder === "") {
folder = undefined;
}
const workspace = await resolveWorkspace(opts);
await requireLogin(opts);
opts = await mergeConfigWithConfigFile(opts);
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(false);
const codebases = await listSyncCodebases(opts);
const ignore = await ignoreF(opts);
// --schema-only implies skipping flows and apps (they only have locks, no schemas)
const skipScripts = opts.skipScripts ?? false;
const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false;
const skipApps = opts.skipApps ?? opts.schemaOnly ?? false;
const checking: string[] = [];
if (!skipScripts) checking.push("scripts");
if (!skipFlows) checking.push("flows");
if (!skipApps) checking.push("apps");
if (checking.length === 0) {
log.info(colors.yellow("Nothing to check (all types skipped)"));
return;
}
log.info(`Checking ${checking.join(", ")}...`);
// Build dependency tree for relative import tracking
const tree = new DoubleLinkedDependencyTree();
tree.setWorkspaceDeps(rawWorkspaceDependencies);
// === Collect stale scripts ===
if (!skipScripts) {
// TODO: run elementsToMap only once but for all runnable types.
const scriptElems = await elementsToMap(
await FSFSElement(process.cwd(), codebases, false),
(p, isD) => {
return (
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
ignore(p, isD) ||
isFolderResourcePathAnyFormat(p) ||
(isScriptModulePath(p) && !isModuleEntryPoint(p))
);
},
false,
{}
);
for (const e of Object.keys(scriptElems)) {
await generateScriptMetadataInternal(
e,
workspace,
opts,
true, // dryRun - populate tree
true, // noStaleMessage
rawWorkspaceDependencies,
codebases,
false,
false, // legacyBehaviour
tree
);
}
}
// === Collect stale flows ===
if (!skipFlows) {
const flowElems = Object.keys(
await elementsToMap(
await FSFSElement(process.cwd(), [], true),
(p, isD) => {
return (
ignore(p, isD) ||
(!isD &&
!p.endsWith(SEP + "flow.yaml") &&
!p.endsWith(SEP + "flow.json"))
);
},
false,
{}
)
).map((x) => x.substring(0, x.lastIndexOf(SEP)));
for (const flowFolder of flowElems) {
await generateFlowLockInternal(
flowFolder,
true, // dryRun - populate tree
workspace,
opts,
false,
true, // noStaleMessage
false, // legacyBehaviour
tree
);
}
}
// === Collect stale apps ===
if (!skipApps) {
const elems = await elementsToMap(
await FSFSElement(process.cwd(), [], true),
(p, isD) => {
return (
ignore(p, isD) ||
(!isD &&
!p.endsWith(SEP + "raw_app.yaml") &&
!p.endsWith(SEP + "app.yaml"))
);
},
false,
{}
);
const rawAppFolders = getAppFolders(elems, "raw_app.yaml");
const appFolders = getAppFolders(elems, "app.yaml");
for (const appFolder of rawAppFolders) {
await generateAppLocksInternal(
appFolder,
true, // rawApp
true, // dryRun - populate tree
workspace,
opts,
false,
true, // noStaleMessage
false, // legacyBehaviour
tree
);
}
for (const appFolder of appFolders) {
await generateAppLocksInternal(
appFolder,
false, // rawApp
true, // dryRun - populate tree
workspace,
opts,
false,
true, // noStaleMessage
false, // legacyBehaviour
tree
);
}
}
// === Propagate staleness through imports ===
tree.propagateStaleness();
// Upload stale scripts to temp storage so the backend can resolve relative imports.
// If this fails (e.g. backend is older and doesn't have /raw_temp endpoints),
// degrade gracefully: locks will be generated using deployed script content only.
try {
await uploadScripts(tree, workspace);
} catch (e) {
log.warn(colors.yellow(
`Failed to upload scripts to temp storage (backend may be too old): ${e}. ` +
`Locks will be generated using deployed script versions only — locally modified ` +
`relative imports may not be reflected.`
));
}
// === Populate staleItems from tree ===
const staleItems: StaleItem[] = [];
const seenFolders = new Set<string>();
for (const p of tree.allPaths()) {
const staleReason = tree.getStaleReason(p);
if (!staleReason) continue;
const itemType = tree.getItemType(p)!;
const itemFolder = tree.getFolder(p)!;
if (itemType === "dependencies") {
staleItems.push({ type: itemType, path: p, folder: itemFolder, staleReason });
} else if (itemType === "inline_script") {
// Inline scripts are not listed separately — their parent flow/app is stale via propagation
continue;
} else if (itemType === "script") {
const originalPath = tree.getOriginalPath(p)!;
staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, staleReason });
} else if (!seenFolders.has(itemFolder)) {
// Flows/Apps: one entry per folder (dedupe multiple inline scripts)
seenFolders.add(itemFolder);
const originalPath = tree.getOriginalPath(p)!;
staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, isRawApp: tree.getIsRawApp(p), staleReason });
}
}
// === Filter by folder if specified ===
let filteredItems = staleItems;
if (folder) {
// Normalize to forward slashes (Windows users may use backslashes)
folder = folder.replaceAll("\\", "/");
// Strip trailing slash to match deprecated flow/app handler behavior
if (folder.endsWith("/")) {
folder = folder.substring(0, folder.length - 1);
}
// Strip file extension if user passed a specific file path (e.g. f/test/script.ts)
const folderNoExt = folder.replace(/\.[^/.]+$/, "");
// Check if an item is inside the specified folder
const isInsideFolder = (item: StaleItem) => {
const normalizedFolder = item.folder.replaceAll("\\", "/");
const normalizedPath = item.path.replaceAll("\\", "/");
return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/")
|| normalizedPath === folder || normalizedPath === folderNoExt;
};
const isPathInFolder = (p: string) => p.startsWith(folder + "/") || p === folder || p === folderNoExt;
// Check if a tree path or any of its transitive deps is inside the folder
const touchesFolder = (treePath: string) => {
if (isPathInFolder(treePath)) return true;
let found = false;
tree.traverseTransitive(treePath, (importPath) => {
if (isPathInFolder(importPath)) {
found = true;
return true; // stop early
}
});
return found;
};
const isRelevant = (item: StaleItem) => {
if (isInsideFolder(item)) return true;
if (item.type === "dependencies") return true;
const treePath = (item.type === "script"
? item.path.replace(/\.[^/.]+$/, "")
: item.folder).replaceAll("\\", "/");
return touchesFolder(treePath);
};
if (opts.strictFolderBoundaries) {
// Strict mode: only items inside the folder
filteredItems = staleItems.filter(isInsideFolder);
// Warn about stale items outside the folder that would be included by default
const excludedStale = staleItems.filter((item) => !isInsideFolder(item) && isRelevant(item) && item.type !== "dependencies");
for (const item of excludedStale) {
const normalizedPath = item.path.replaceAll("\\", "/");
log.warn(colors.yellow(
`Warning: ${normalizedPath} depends on something inside "${folder}" but is outside it — skipped due to --strict-folder-boundaries. Next generate-metadata will not detect it as stale.`
));
}
} else {
// Default: include items inside the folder and any stale importers that transitively depend on it
filteredItems = staleItems.filter(isRelevant);
}
}
// === Show stale items and confirm ===
if (filteredItems.length === 0) {
log.info(colors.green("All metadata up-to-date"));
return;
}
// Group items by type for display
const scripts = filteredItems.filter((i) => i.type === "script");
const flows = filteredItems.filter((i) => i.type === "flow");
const apps = filteredItems.filter((i) => i.type === "app");
const deps = filteredItems.filter((i) => i.type === "dependencies");
log.info("");
log.info(`Found ${colors.bold(String(filteredItems.length))} item(s) with stale metadata:`);
const printItems = (label: string, items: StaleItem[]) => {
if (items.length === 0) return;
log.info(` ${label} (${items.length}):`);
for (const item of items) {
const reason = item.staleReason ? colors.dim(colors.white(` — ${item.staleReason}`)) : "";
log.info(` ~ ${item.path}` + reason);
}
};
printItems("Workspace dependencies", deps);
printItems("Scripts", scripts);
printItems("Flows", flows);
printItems("Apps", apps);
if (opts.dryRun) {
return;
}
log.info("");
if (
!opts.yes &&
!(await Confirm.prompt({
message: "Update metadata?",
default: true,
}))
) {
return;
}
log.info("");
// === Process all stale items with progress counter ===
const mismatchedWorkspaceDeps = tree.getMismatchedWorkspaceDeps();
const total = filteredItems.length - deps.length;
const maxWidth = `[${total}/${total}]`.length;
let current = 0;
const formatProgress = (n: number) => {
return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " ")));
};
// Process scripts
for (const item of scripts) {
current++;
log.info(`${formatProgress(current)} script ${item.path}`);
await generateScriptMetadataInternal(
item.path, // originalPath with extension
workspace,
opts,
false, // dryRun
true, // noStaleMessage
mismatchedWorkspaceDeps,
codebases,
false,
false, // legacyBehaviour
tree
);
}
// Process flows
for (const item of flows) {
current++;
const result = await generateFlowLockInternal(
item.folder.replaceAll("/", SEP),
false, // dryRun
workspace,
opts,
false,
true, // noStaleMessage
false, // legacyBehaviour
tree
);
const flowResult = result as FlowLocksResult | undefined;
const scriptsInfo = flowResult?.updatedScripts?.length
? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`))
: "";
log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`);
}
// Process apps
for (const item of apps) {
current++;
const result = await generateAppLocksInternal(
item.folder.replaceAll("/", SEP),
item.isRawApp!, // rawApp
false, // dryRun
workspace,
opts,
false,
true, // noStaleMessage
false, // legacyBehaviour
tree
);
const appResult = result as AppLocksResult | undefined;
const scriptsInfo = appResult?.updatedScripts?.length
? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`))
: "";
log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`);
}
// Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped)
const allStaleDeps = staleItems.filter((i) => i.type === "dependencies");
await tree.persistDepsHashes(allStaleDeps.map((d) => d.path));
log.info("");
log.info(`Done. Updated ${colors.bold(String(total))} item(s).`);
}
const command = new Command()
.description("Generate metadata (locks, schemas) for all scripts, flows, and apps")
.arguments("[folder:string]")
.option("--yes", "Skip confirmation prompt")
.option("--dry-run", "Show what would be updated without making changes")
.option("--lock-only", "Re-generate only the lock files")
.option("--schema-only", "Re-generate only script schemas (skips flows and apps)")
.option("--skip-scripts", "Skip processing scripts")
.option("--skip-flows", "Skip processing flows")
.option("--skip-apps", "Skip processing apps")
.option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)")
.option(
"-i --includes <patterns:file[]>",
"Comma separated patterns to specify which files to include"
)
.option(
"-e --excludes <patterns:file[]>",
"Comma separated patterns to specify which files to exclude"
)
.action(generateMetadata as any);
export default command;
+24
View File
@@ -252,6 +252,16 @@ async function initAction(opts: InitOptions) {
}
}
// Read nonDottedPaths from config to specialize generated skills
let nonDottedPaths = true; // default for new inits
try {
const { readConfigFile } = await import("../../core/conf.ts");
const config = await readConfigFile();
nonDottedPaths = config.nonDottedPaths ?? true;
} catch {
// If config can't be read, use default
}
// Create guidance files (AGENTS.md, CLAUDE.md, and Claude skills)
try {
// Generate skills reference section for AGENTS.md
@@ -290,6 +300,20 @@ async function initAction(opts: InitOptions) {
let skillContent = SKILL_CONTENT[skill.name];
if (skillContent) {
// Replace placeholders with actual suffixes based on nonDottedPaths
if (nonDottedPaths) {
skillContent = skillContent
.replaceAll("{{FLOW_SUFFIX}}", "__flow")
.replaceAll("{{APP_SUFFIX}}", "__app")
.replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app")
.replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`).");
} else {
skillContent = skillContent
.replaceAll("{{FLOW_SUFFIX}}", ".flow")
.replaceAll("{{APP_SUFFIX}}", ".app")
.replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app")
.replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`).");
}
// Check if this skill has schemas that need to be appended
const schemaMappings = SCHEMA_MAPPINGS[skill.name];
if (schemaMappings && schemaMappings.length > 0) {
+200 -20
View File
@@ -1,7 +1,7 @@
import { GlobalOptions } from "../../types.ts";
import { requireLogin } from "../../core/auth.ts";
import { resolveWorkspace, validatePath } from "../../core/context.ts";
import { readFile, writeFile, stat } from "node:fs/promises";
import { readFile, writeFile, stat, mkdir } from "node:fs/promises";
import { Buffer } from "node:buffer";
import { colors } from "@cliffy/ansi/colors";
import { Command } from "@cliffy/command";
@@ -9,6 +9,7 @@ import { Confirm } from "@cliffy/prompt/confirm";
import { Table } from "@cliffy/table";
import * as log from "../../core/log.ts";
import { sep as SEP } from "node:path";
import * as path from "node:path";
import { stringify as yamlStringify } from "yaml";
import { deepEqual } from "../../utils/utils.ts";
import * as wmill from "../../../gen/services.gen.ts";
@@ -51,7 +52,7 @@ import fs from "node:fs";
import { createTarBlob, type TarEntry } from "../../utils/tar.ts";
import { execSync } from "node:child_process";
import { NewScript, Script } from "../../../gen/types.gen.ts";
import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts";
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
import { resolvePermissionedAsRule } from "../../core/permissioned_as.ts";
import {
@@ -60,6 +61,12 @@ import {
isFlowInlineScriptPath as isFlowInlineScriptPathInternal,
isFlowPath,
isAppPath,
isScriptModulePath,
buildModuleFolderPath,
getModuleFolderSuffix,
isModuleEntryPoint,
getScriptBasePathFromModulePath,
isRawAppPath,
} from "../../utils/resource_folders.ts";
export interface ScriptFile {
@@ -125,7 +132,7 @@ async function push(opts: PushOptions, filePath: string) {
[],
undefined,
opts,
await getRawWorkspaceDependencies(),
await getRawWorkspaceDependencies(true),
codebases
);
log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
@@ -191,11 +198,17 @@ export async function handleScriptMetadata(
opts: GlobalOptions,
permissionedAsContext?: PermissionedAsContext
): Promise<boolean> {
if (
path.endsWith(".script.json") ||
// Flat layout: my_script.script.yaml
const isFlatMeta = path.endsWith(".script.json") ||
path.endsWith(".script.yaml") ||
path.endsWith(".script.lock")
) {
path.endsWith(".script.lock");
// Folder layout: my_script__mod/script.yaml
const isFolderMeta = !isFlatMeta && isScriptModulePath(path) && (
path.endsWith("/script.yaml") ||
path.endsWith("/script.json") ||
path.endsWith("/script.lock")
);
if (isFlatMeta || isFolderMeta) {
const contentPath = await findContentFile(path);
return handleFile(
contentPath,
@@ -230,10 +243,13 @@ export async function handleFile(
codebases: SyncCodebase[],
permissionedAsContext?: PermissionedAsContext
): Promise<boolean> {
// Detect module entry point: e.g., my_script__mod/script.ts
const moduleEntryPoint = isModuleEntryPoint(path);
if (
!isAppInlineScriptPath(path) &&
!isFlowInlineScriptPath(path) &&
!isRawAppBackendPath(path) &&
(!isScriptModulePath(path) || moduleEntryPoint) &&
exts.some((exts) => path.endsWith(exts))
) {
if (alreadySynced.includes(path)) {
@@ -242,9 +258,9 @@ export async function handleFile(
log.debug(`Processing local script ${path}`);
alreadySynced.push(path);
const remotePath = path
.substring(0, path.indexOf("."))
.replaceAll(SEP, "/");
const remotePath = moduleEntryPoint
? getScriptBasePathFromModulePath(path)!.replaceAll(SEP, "/")
: path.substring(0, path.indexOf(".")).replaceAll(SEP, "/");
const language = inferContentTypeFromFilePath(path, opts?.defaultTs);
@@ -396,6 +412,13 @@ export async function handleFile(
typed.codebase = await codebase.getDigest(forceTar);
}
// Scan for modules: folder layout (entry point inside __mod/) or flat layout
const scriptBasePath = moduleEntryPoint
? getScriptBasePathFromModulePath(path)!
: path.substring(0, path.indexOf("."));
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, moduleEntryPoint);
const requestBodyCommon: NewScript = {
content,
description: typed?.description ?? "",
@@ -414,7 +437,6 @@ export async function handleFile(
deployment_message: message,
restart_unless_cancelled: typed?.restart_unless_cancelled,
visible_to_runner_only: typed?.visible_to_runner_only,
no_main_func: typed?.no_main_func,
has_preprocessor: typed?.has_preprocessor,
priority: typed?.priority,
concurrency_key: typed?.concurrency_key,
@@ -424,6 +446,7 @@ export async function handleFile(
timeout: typed?.timeout,
on_behalf_of_email: typed?.on_behalf_of_email,
envs: typed?.envs,
modules: modules,
};
// Add preserve flags for permissioned_as
@@ -474,7 +497,6 @@ export async function handleFile(
Boolean(remote.restart_unless_cancelled) &&
Boolean(typed.visible_to_runner_only) ==
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
Boolean(typed.has_preprocessor) ==
Boolean(remote.has_preprocessor) &&
typed.priority == Boolean(remote.priority) &&
@@ -485,7 +507,8 @@ export async function handleFile(
typed.debounce_delay_s == remote["debounce_delay_s"] &&
typed.codebase == remote.codebase &&
typed.on_behalf_of_email == remote.on_behalf_of_email &&
deepEqual(typed.envs, remote.envs))
deepEqual(typed.envs, remote.envs) &&
deepEqual(modules ?? null, remote.modules ?? null))
) {
log.info(colors.green(`Script ${remotePath} is up to date`));
return true;
@@ -531,6 +554,135 @@ export async function handleFile(
return false;
}
/**
* Read module files from a __mod/ directory on disk.
* Returns the modules record for the API, or undefined if no module folder exists.
*/
export async function readModulesFromDisk(
moduleFolderPath: string,
defaultTs: "bun" | "deno" | undefined,
folderLayout: boolean = false,
): Promise<Record<string, ScriptModule> | undefined> {
if (!fs.existsSync(moduleFolderPath) || !fs.statSync(moduleFolderPath).isDirectory()) {
return undefined;
}
const modules: Record<string, ScriptModule> = {};
// In folder layout mode, skip the entry point files (script.*, script.yaml, etc.)
const isEntryPointFile = (name: string, isTopLevel: boolean) => {
if (!folderLayout || !isTopLevel) return false;
return (
name.startsWith("script.") ||
name === "script.lock" ||
name === "script.yaml" ||
name === "script.json"
);
};
function readDir(dirPath: string, relPrefix: string) {
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
const isTopLevel = relPrefix === "";
if (entry.isDirectory()) {
readDir(fullPath, relPath);
} else if (entry.isFile() && !entry.name.endsWith(".lock") && !isEntryPointFile(entry.name, isTopLevel)) {
// Skip lock files — they're handled as the `lock` field on ScriptModule
if (exts.some((ext) => entry.name.endsWith(ext))) {
const content = fs.readFileSync(fullPath, "utf-8");
const language = inferContentTypeFromFilePath(entry.name, defaultTs);
// Check for an accompanying lock file (helper.lock)
const baseName = entry.name.replace(/\.[^.]+$/, '');
const lockPath = path.join(dirPath, baseName + ".lock");
let lock: string | undefined;
if (fs.existsSync(lockPath)) {
lock = fs.readFileSync(lockPath, "utf-8");
}
modules[relPath] = {
content,
language: language as ScriptModule["language"],
lock: lock ?? undefined,
};
}
}
}
}
readDir(moduleFolderPath, "");
if (Object.keys(modules).length === 0) {
return undefined;
}
log.debug(`Found ${Object.keys(modules).length} module(s) in ${moduleFolderPath}`);
return modules;
}
/**
* Write module files to a __mod/ directory on disk during pull.
*/
export async function writeModulesToDisk(
moduleFolderPath: string,
modules: Record<string, ScriptModule>,
defaultTs: "bun" | "deno" | undefined
): Promise<void> {
// Ensure the module folder exists
fs.mkdirSync(moduleFolderPath, { recursive: true });
// Clean up stale module files that are no longer in the modules map
const expectedFiles = new Set<string>();
for (const [relPath, mod] of Object.entries(modules)) {
expectedFiles.add(relPath);
if (mod.lock) {
expectedFiles.add(relPath.replace(/\.[^.]+$/, '') + ".lock");
}
}
function cleanDir(dirPath: string, relPrefix: string) {
if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) return;
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
if (entry.isDirectory()) {
cleanDir(path.join(dirPath, entry.name), relPath);
// Remove empty directories after cleaning
try {
const remaining = fs.readdirSync(path.join(dirPath, entry.name));
if (remaining.length === 0) {
fs.rmdirSync(path.join(dirPath, entry.name));
}
} catch {}
} else if (!expectedFiles.has(relPath)) {
fs.unlinkSync(path.join(dirPath, entry.name));
}
}
}
cleanDir(moduleFolderPath, "");
for (const [relPath, mod] of Object.entries(modules)) {
const fullPath = path.join(moduleFolderPath, relPath);
const dir = path.dirname(fullPath);
fs.mkdirSync(dir, { recursive: true });
// Write the module content
fs.writeFileSync(fullPath, mod.content, "utf-8");
// Write the lock file if present
if (mod.lock) {
const baseName = relPath.replace(/\.[^.]+$/, '');
const lockPath = path.join(moduleFolderPath, baseName + ".lock");
const lockDir = path.dirname(lockPath);
fs.mkdirSync(lockDir, { recursive: true });
fs.writeFileSync(lockPath, mod.lock, "utf-8");
}
}
}
async function createScript(
bundleContent: string | Blob | undefined,
workspaceId: string,
@@ -584,7 +736,12 @@ async function createScript(
}
export async function findContentFile(filePath: string) {
const candidates = filePath.endsWith("script.json")
// Folder layout: __mod/script.yaml -> __mod/script.ts
const isModuleFolderMeta =
filePath.endsWith("/script.yaml") || filePath.endsWith("/script.json") || filePath.endsWith("/script.lock");
const candidates = isModuleFolderMeta
? exts.map((x) => filePath.replace(/\/script\.(yaml|json|lock)$/, "/script" + x))
: filePath.endsWith("script.json")
? exts.map((x) => filePath.replace(".script.json", x))
: filePath.endsWith("script.lock")
? exts.map((x) => filePath.replace(".script.lock", x))
@@ -937,16 +1094,22 @@ async function get(opts: GlobalOptions & { json?: boolean }, path: string) {
}
}
const languageAliases: Record<string, ScriptLanguage> = {
python: "python3",
};
async function bootstrap(
opts: GlobalOptions & { summary: string; description: string },
scriptPath: string,
language: ScriptLanguage
language: ScriptLanguage | string
) {
if (!validatePath(scriptPath)) {
return;
}
const scriptInitialCode = scriptBootstrapCode[language];
const resolvedLanguage = (languageAliases[language] ?? language) as ScriptLanguage;
const scriptInitialCode = scriptBootstrapCode[resolvedLanguage];
if (scriptInitialCode === undefined) {
throw new Error("Language unknown");
}
@@ -954,7 +1117,7 @@ async function bootstrap(
const config = await readConfigFile();
const extension = filePathExtensionFromContentType(
language,
resolvedLanguage,
config.defaultTs
);
const scriptCodeFileFullPath = scriptPath + extension;
@@ -986,6 +1149,9 @@ async function bootstrap(
yamlOptions
);
const parentDir = path.dirname(scriptCodeFileFullPath);
await mkdir(parentDir, { recursive: true });
await writeFile(scriptCodeFileFullPath, scriptInitialCode, {
flag: 'wx', encoding: 'utf-8',
});
@@ -1003,7 +1169,7 @@ export type GlobalDeps = Map<
Record<string, string>
>;
async function generateMetadata(
export async function generateMetadata(
opts: GlobalOptions & {
lockOnly?: boolean;
schemaOnly?: boolean;
@@ -1011,6 +1177,9 @@ async function generateMetadata(
} & SyncOptions,
scriptPath: string | undefined
) {
log.warn(
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
);
log.info(
"This command only works for workspace scripts, for flows inline scripts use `wmill flow generate-locks`"
);
@@ -1026,7 +1195,7 @@ async function generateMetadata(
opts = await mergeConfigWithConfigFile(opts);
const codebases = await listSyncCodebases(opts);
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
if (scriptPath) {
// read script metadata file
await generateScriptMetadataInternal(
@@ -1049,7 +1218,10 @@ async function generateMetadata(
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
ignore(p, isD) ||
isFlowPath(p) ||
isAppPath(p)
isAppPath(p) ||
isRawAppPath(p) ||
// Skip module helper files; only entry points (script.{ext}) are processed
(isScriptModulePath(p) && !isModuleEntryPoint(p))
);
},
false,
@@ -1138,6 +1310,13 @@ async function preview(
const content = await readFile(filePath, "utf-8");
const input = opts.data ? await resolve(opts.data) : {};
// Read modules from __mod/ folder if present
const isFolderLayout = isModuleEntryPoint(filePath);
const moduleFolderPath = isFolderLayout
? path.dirname(filePath)
: filePath.substring(0, filePath.indexOf(".")) + getModuleFolderSuffix();
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, isFolderLayout);
// Check if this is a codebase script
const codebase =
language == "bun" ? findCodebase(filePath, codebases) : undefined;
@@ -1297,6 +1476,7 @@ async function preview(
path: filePath.substring(0, filePath.indexOf(".")).replaceAll(SEP, "/"),
args: input,
language: language as any,
modules: modules ?? undefined,
},
});
+282 -17
View File
@@ -31,6 +31,7 @@ import {
findResourceFile,
handleScriptMetadata,
removeExtensionToPath,
filePathExtensionFromContentType,
} from "../script/script.ts";
import { handleFile } from "../script/script.ts";
@@ -69,7 +70,7 @@ import {
readLockfile,
workspaceDependenciesPathToLanguageAndFilename,
} from "../../utils/metadata.ts";
import { OpenFlow, NativeServiceName } from "../../../gen/types.gen.ts";
import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts";
import { pushResource } from "../resource/resource.ts";
import {
newPathAssigner,
@@ -103,6 +104,10 @@ import {
getFolderSuffix,
getFolderSuffixWithSep,
getNonDottedPaths,
isScriptModulePath,
getModuleFolderSuffix,
isModuleEntryPoint,
getScriptBasePathFromModulePath,
} from "../../utils/resource_folders.ts";
// Merge CLI options with effective settings, preserving CLI flags as overrides
@@ -544,6 +549,29 @@ function ZipFSElement(
ignoreCodebaseChanges: boolean,
stripOnBehalfOf: boolean,
): DynFSElement {
// Pre-scan: find zip base paths of scripts that have modules.
// These scripts use the folder layout: {basePath}__mod/script.{ext}
let _moduleScriptPaths: Set<string> | null = null;
async function getModuleScriptPaths(): Promise<Set<string>> {
if (_moduleScriptPaths === null) {
_moduleScriptPaths = new Set();
for (const filename in zip.files) {
if (filename.endsWith(".script.json") && !zip.files[filename].dir) {
try {
const content = await zip.files[filename].async("text");
const parsed = JSON.parse(content);
if (parsed.modules && Object.keys(parsed.modules).length > 0) {
_moduleScriptPaths.add(
filename.slice(0, -".script.json".length)
);
}
} catch {}
}
}
}
return _moduleScriptPaths;
}
async function _internal_file(
p: string,
f: JSZip.JSZipObject,
@@ -585,7 +613,22 @@ function ZipFSElement(
}
}
const finalPath = transformPath();
let finalPath = transformPath();
// Redirect content files for scripts with modules into __mod/ folder
if (kind == "other" && exts.some((ext) => p.endsWith(ext))) {
const normalizedP = p.replace(/^\.[\\/]/, "");
const moduleScripts = await getModuleScriptPaths();
for (const basePath of moduleScripts) {
if (normalizedP.startsWith(basePath + ".")) {
const ext = normalizedP.slice(basePath.length); // e.g., ".ts", ".py"
const dir = path.dirname(finalPath);
const base = path.basename(basePath);
finalPath = path.join(dir, base + getModuleFolderSuffix(), "script" + ext);
break;
}
}
}
const r = [
{
@@ -602,14 +645,35 @@ function ZipFSElement(
}
let inlineScripts;
try {
const assigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() });
inlineScripts = extractInlineScriptsForFlows(
flow.value.modules as any,
{},
SEP,
defaultTs,
undefined, // pathAssigner - let it create one
assigner,
{ skipInlineScriptSuffix: getNonDottedPaths() },
);
if (flow.value.failure_module) {
inlineScripts.push(...extractInlineScriptsForFlows(
[flow.value.failure_module],
{},
SEP,
defaultTs,
assigner,
{ skipInlineScriptSuffix: getNonDottedPaths() },
));
}
if (flow.value.preprocessor_module) {
inlineScripts.push(...extractInlineScriptsForFlows(
[flow.value.preprocessor_module],
{},
SEP,
defaultTs,
assigner,
{ skipInlineScriptSuffix: getNonDottedPaths() },
));
}
} catch (error) {
log.error(
`Failed to extract inline scripts for flow at path: ${p}`,
@@ -886,15 +950,23 @@ function ZipFSElement(
log.error(`Failed to parse script.yaml at path: ${p}`);
throw error;
}
const hasModules = parsed["modules"] && Object.keys(parsed["modules"]).length > 0;
if (
parsed["lock"] &&
parsed["lock"] != "" &&
parsed["codebase"] == undefined
) {
parsed["lock"] =
"!inline " +
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
".lock";
if (hasModules) {
// Lock lives inside __mod/ folder as script.lock
const scriptBase = removeSuffix(removeSuffix(p.replaceAll(SEP, "/"), ".json"), ".script");
parsed["lock"] =
"!inline " + scriptBase + getModuleFolderSuffix() + "/script.lock";
} else {
parsed["lock"] =
"!inline " +
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
".lock";
}
} else if (parsed["lock"] == "") {
parsed["lock"] = "";
} else {
@@ -906,6 +978,8 @@ function ZipFSElement(
if (stripOnBehalfOf) {
delete parsed["on_behalf_of_email"];
}
// Modules are stored as files in __mod/ folder, not in metadata
delete parsed["modules"];
return useYaml
? yamlStringify(parsed, yamlOptions)
: JSON.stringify(parsed, null, 2);
@@ -965,16 +1039,71 @@ function ZipFSElement(
throw error;
}
const lock = parsed["lock"];
const scriptModules: Record<string, ScriptModule> | undefined = parsed["modules"];
const hasModules = scriptModules && Object.keys(scriptModules).length > 0;
// Compute base path and module folder
const metaExt = useYaml ? ".yaml" : ".json";
const scriptBasePath = removeSuffix(
removeSuffix(finalPath, metaExt),
".script"
);
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
if (hasModules) {
// Redirect metadata into __mod/script.yaml
r[0].path = path.join(moduleFolderPath, "script" + metaExt);
}
if (lock && lock != "") {
r.push({
isDirectory: false,
path: removeSuffix(finalPath, ".json") + ".lock",
path: hasModules
? path.join(moduleFolderPath, "script.lock")
: removeSuffix(finalPath, metaExt) + ".lock",
async *getChildren() {},
async getContentText() {
return lock;
},
});
}
// Extract script modules into __mod/ folder
if (hasModules) {
r.push({
isDirectory: true,
path: moduleFolderPath,
async *getChildren() {
for (const [relPath, mod] of Object.entries(scriptModules!)) {
// Yield the module content file
yield {
isDirectory: false,
path: path.join(moduleFolderPath, relPath),
async *getChildren() {},
async getContentText() {
return mod.content;
},
};
// Yield the module lock file if present
if (mod.lock) {
const baseName = relPath.replace(/\.[^.]+$/, '');
yield {
isDirectory: false,
path: path.join(moduleFolderPath, baseName + ".lock"),
async *getChildren() {},
async getContentText() {
return mod.lock!;
},
};
}
}
},
async getContentText() {
throw new Error("Cannot get content of directory");
},
});
}
}
if (kind == "resource") {
const content = await f.async("text");
@@ -1150,6 +1279,12 @@ export async function elementsToMap(
continue;
}
const path = entry.path;
// Include module files in the map so they're compared for changes,
// but they're pushed as part of their parent script via handleFile
if (isScriptModulePath(path)) {
map[path] = await entry.getContentText();
continue;
}
if (
!isFileResource(path) &&
!isFilesetResource(path) &&
@@ -1596,6 +1731,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
);
}
// Files inside __mod/ folders are script module files — always valid wmill files
if (isScriptModulePath(p)) {
return false;
}
try {
const typ = getTypeStrFromPath(p);
if (
@@ -1740,6 +1880,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
if (!tracker.rawApps.includes(folder)) {
tracker.rawApps.push(folder);
}
} else if (isScriptModulePath(p)) {
if (isModuleEntryPoint(p)) {
// Entry point (e.g. __mod/script.ts) IS the parent script content file
if (!tracker.scripts.includes(p)) {
tracker.scripts.push(p);
}
} else {
// Module file changed — find the parent script content file
const moduleSuffix = getModuleFolderSuffix() + "/";
const idx = p.indexOf(moduleSuffix);
if (idx !== -1) {
const scriptBasePath = p.substring(0, idx);
// Try folder layout first: __mod/script.{ext}
try {
const contentPath = await findContentFile(scriptBasePath + getModuleFolderSuffix() + "/script.yaml");
if (contentPath && !tracker.scripts.includes(contentPath)) {
tracker.scripts.push(contentPath);
}
} catch {
// Fall back to flat layout: scriptBasePath.script.yaml
try {
const contentPath = await findContentFile(scriptBasePath + ".script.yaml");
if (contentPath && !tracker.scripts.includes(contentPath)) {
tracker.scripts.push(contentPath);
}
} catch {
// ignore — content file not found
}
}
}
}
} else {
if (!tracker.scripts.includes(p)) {
tracker.scripts.push(p);
@@ -1772,6 +1943,61 @@ async function buildTracker(changes: Change[]) {
return tracker;
}
/**
* When a module file changes, find and push the parent script.
* The parent script's handleFile will read the __mod/ folder and include all modules.
*/
async function pushParentScriptForModule(
modulePath: string,
workspace: Workspace,
alreadySynced: string[],
message: string | undefined,
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[],
): Promise<void> {
const moduleSuffix = getModuleFolderSuffix() + "/";
const idx = modulePath.indexOf(moduleSuffix);
if (idx === -1) return;
const scriptBasePath = modulePath.substring(0, idx);
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
// Try folder layout first: look for script.{ext} inside __mod/
try {
const entryPoint = await findContentFile(moduleFolderPath + "/script.yaml");
if (entryPoint) {
await handleFile(
entryPoint,
workspace,
alreadySynced,
message,
opts,
rawWorkspaceDependencies,
codebases,
);
return;
}
} catch {}
// Fall back to flat layout: look for content file alongside __mod/
try {
const contentPath = await findContentFile(scriptBasePath + ".script.yaml");
if (contentPath) {
await handleFile(
contentPath,
workspace,
alreadySynced,
message,
opts,
rawWorkspaceDependencies,
codebases,
);
}
} catch {
log.debug(`Could not find parent script for module: ${modulePath}`);
}
}
export async function pull(
opts: GlobalOptions &
SyncOptions & { repository?: string; promotion?: string; branch?: string },
@@ -2072,7 +2298,7 @@ export async function pull(
const tracker: ChangeTracker = await buildTracker(changes);
const rawWorkspaceDependencies: Record<string, string> =
await getRawWorkspaceDependencies();
await getRawWorkspaceDependencies(true);
for (const change of tracker.scripts) {
await generateScriptMetadataInternal(
@@ -2404,7 +2630,7 @@ export async function push(
false, // els1 (local) is not the remote source
);
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
const tracker: ChangeTracker = await buildTracker(changes);
@@ -2450,7 +2676,7 @@ export async function push(
true,
);
if (stale) {
staleFlows.push(stale);
staleFlows.push(stale as string);
}
}
@@ -2475,7 +2701,7 @@ export async function push(
true,
);
if (stale) {
staleApps.push(stale);
staleApps.push(stale as string);
}
}
@@ -2490,7 +2716,7 @@ export async function push(
true,
);
if (stale) {
staleApps.push(stale);
staleApps.push(stale as string);
}
}
@@ -2733,6 +2959,21 @@ export async function push(
await writeFile(stateTarget, change.after, "utf-8");
}
continue;
} else if (isScriptModulePath(change.path)) {
// Module file changed — push the parent script
await pushParentScriptForModule(
change.path,
workspace,
alreadySynced,
opts.message,
opts,
rawWorkspaceDependencies,
codebases,
);
if (stateTarget) {
await writeFile(stateTarget, change.after, "utf-8");
}
continue;
}
if (stateTarget) {
await mkdir(path.dirname(stateTarget), { recursive: true });
@@ -2859,6 +3100,17 @@ export async function push(
)
) {
continue;
} else if (isScriptModulePath(change.path)) {
await pushParentScriptForModule(
change.path,
workspace,
alreadySynced,
opts.message,
opts,
rawWorkspaceDependencies,
codebases,
);
continue;
}
if (stateTarget) {
await mkdir(path.dirname(stateTarget), { recursive: true });
@@ -2901,6 +3153,19 @@ export async function push(
if (change.path.endsWith(".lock")) {
continue;
}
if (isScriptModulePath(change.path)) {
// Module file deleted — push the parent script (which will now have fewer modules)
await pushParentScriptForModule(
change.path,
workspace,
alreadySynced,
opts.message,
opts,
rawWorkspaceDependencies,
codebases,
);
continue;
}
const typ = getTypeStrFromPath(change.path);
if (typ == "script") {
@@ -3276,8 +3541,8 @@ const command = new Command()
"Use promotionOverrides from the specified branch instead of regular overrides",
)
.option(
"--branch <branch:string>",
"Override the current git branch (works even outside a git repository)",
"--branch, --env <branch:string>",
"Override the current git branch/environment (works even outside a git repository)",
)
.action(pull as any)
.command("push")
@@ -3332,8 +3597,8 @@ const command = new Command()
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
)
.option(
"--branch <branch:string>",
"Override the current git branch (works even outside a git repository)",
"--branch, --env <branch:string>",
"Override the current git branch/environment (works even outside a git repository)",
)
.option("--lint", "Run lint validation before pushing")
.option(
+4
View File
@@ -268,6 +268,7 @@ export async function pushNativeTrigger(
is_flow: result.is_flow,
service_config: result.service_config,
error: result.error,
summary: result.summary,
};
log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`);
} catch {
@@ -280,6 +281,7 @@ export async function pushNativeTrigger(
script_path: localTrigger.script_path,
is_flow: localTrigger.is_flow,
service_config: localTrigger.service_config,
summary: localTrigger.summary,
};
if (remoteTrigger) {
@@ -288,11 +290,13 @@ export async function pushNativeTrigger(
script_path: localTrigger.script_path,
is_flow: localTrigger.is_flow,
service_config: localTrigger.service_config,
summary: localTrigger.summary,
};
const remoteCompare = {
script_path: remoteTrigger.script_path,
is_flow: remoteTrigger.is_flow,
service_config: remoteTrigger.service_config,
summary: remoteTrigger.summary,
};
if (isSuperset(localCompare, remoteCompare)) {
+2 -2
View File
@@ -568,11 +568,11 @@ const command = new Command()
.action(listRemote as any)
.command("bind")
.description("Bind the current Git branch to the active workspace")
.option("--branch <branch:string>", "Specify branch (defaults to current)")
.option("--branch, --env <branch:string>", "Specify branch/environment (defaults to current)")
.action((opts) => bind(opts as any, true))
.command("unbind")
.description("Remove workspace binding from the current Git branch")
.option("--branch <branch:string>", "Specify branch (defaults to current)")
.option("--branch, --env <branch:string>", "Specify branch/environment (defaults to current)")
.action((opts) => bind(opts as any, false))
.command("fork")
.description("Create a forked workspace")
+14
View File
@@ -85,6 +85,8 @@ export interface SyncOptions {
};
};
};
// Alias for gitBranches - for users who prefer environment-based terminology
environments?: SyncOptions["gitBranches"];
// Legacy field - deprecated, use gitBranches instead
git_branches?: {
commonSpecificItems?: {
@@ -243,6 +245,18 @@ export async function readConfigFile(): Promise<SyncOptions> {
}
}
// Handle environments -> gitBranches alias (permanent alias, not a deprecation)
if (conf && "environments" in conf) {
if (!conf.gitBranches) {
conf.gitBranches = conf.environments as any;
} else {
log.warn(
"⚠️ Both 'environments' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'environments'."
);
}
delete (conf as any).environments;
}
// Handle git_branches to gitBranches migration
if (conf && "git_branches" in conf) {
if (!conf.gitBranches) {
+11
View File
@@ -53,6 +53,7 @@ export interface SimplifiedSettings {
mute_critical_alerts?: boolean;
color?: string;
operator_settings?: any;
datatable?: any;
slack_team_id?: string;
slack_name?: string;
slack_command_script?: string;
@@ -100,6 +101,7 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings {
if (settings.mute_critical_alerts !== undefined) result.mute_critical_alerts = settings.mute_critical_alerts;
if (settings.color !== undefined) result.color = settings.color;
if (settings.operator_settings !== undefined) result.operator_settings = settings.operator_settings;
if (settings.datatable !== undefined) result.datatable = settings.datatable;
if (settings.slack_team_id !== undefined) result.slack_team_id = settings.slack_team_id;
if (settings.slack_name !== undefined) result.slack_name = settings.slack_name;
if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script;
@@ -192,6 +194,7 @@ export async function pushWorkspaceSettings(
mute_critical_alerts: remoteSettings.mute_critical_alerts,
color: remoteSettings.color,
operator_settings: remoteSettings.operator_settings,
datatable: remoteSettings.datatable,
slack_team_id: remoteSettings.slack_team_id,
slack_name: remoteSettings.slack_name,
slack_command_script: remoteSettings.slack_command_script,
@@ -382,6 +385,14 @@ export async function pushWorkspaceSettings(
});
}
if (!deepEqual(localSettings.datatable, settings.datatable)) {
log.debug(`Updating datatable config...`);
await wmill.editDataTableConfig({
workspace,
requestBody: { settings: localSettings.datatable ?? { datatables: {} } },
});
}
if (localSettings.slack_command_script != settings.slack_command_script) {
log.debug(`Updating slack command script...`);
await wmill.editSlackCommand({
+1956 -1794
View File
File diff suppressed because one or more lines are too long
+3 -1
View File
@@ -39,6 +39,7 @@ import queues from "./commands/queues/queues.ts";
import dependencies from "./commands/dependencies/dependencies.ts";
import init from "./commands/init/init.ts";
import jobs from "./commands/jobs/jobs.ts";
import generateMetadata from "./commands/generate-metadata/generate-metadata.ts";
import docs from "./commands/docs/docs.ts";
import { fetchVersion } from "./core/context.ts";
@@ -67,7 +68,7 @@ export {
workspaceAdd,
};
export const VERSION = "1.653.0";
export const VERSION = "1.664.0";
// Re-exported from constants.ts to maintain backwards compatibility
export { WM_FORK_PREFIX } from "./core/constants.ts";
@@ -129,6 +130,7 @@ const command = new Command()
.command("queues", queues)
.command("dependencies", dependencies)
.command("jobs", jobs)
.command("generate-metadata", generateMetadata)
.command("docs", docs)
.command("version --version", "Show version information")
.action(async (opts: any) => {
+4
View File
@@ -28,6 +28,7 @@ import {
isRawAppPath,
extractResourceName,
buildFolderPath,
isScriptModulePath,
} from "./utils/resource_folders.ts";
import type { PermissionedAsContext } from "./core/permissioned_as.ts";
@@ -261,6 +262,9 @@ export function getTypeStrFromPath(
| "settings"
| "encryption_key"
| "workspace_dependencies" {
if (isScriptModulePath(p)) {
return "script";
}
if (isFlowPath(p)) {
return "flow";
}
+373
View File
@@ -0,0 +1,373 @@
/**
* Double-linked dependency tree for tracking script imports and propagating staleness.
*/
import { Workspace } from "../commands/workspace/workspace.ts";
import * as wmill from "../../gen/services.gen.ts";
import type { ScriptLang } from "../../gen/types.gen.ts";
import { ScriptLanguage } from "./script_common.ts";
import {
filterWorkspaceDependencies,
generateScriptHash,
checkifMetadataUptodate,
workspaceDependenciesPathToLanguageAndFilename,
updateMetadataGlobalLock,
} from "./metadata.ts";
import { generateHash } from "./utils.ts";
/**
* Diff local scripts against deployed versions, upload only those that differ.
* Only uploaded (mismatched) scripts get contentHash set, so flatten() returns
* temp_script_refs only for scripts the backend can't resolve from deployed versions.
*/
export async function uploadScripts(
tree: DoubleLinkedDependencyTree,
workspace: Workspace
): Promise<void> {
// Split into scripts vs workspace deps and compute SHA256(content) for each
const scriptHashes: Record<string, string> = {};
const workspaceDeps: { path: string; language: ScriptLang; name?: string; hash: string }[] = [];
for (const path of tree.allPaths()) {
const content = tree.getContent(path);
const itemType = tree.getItemType(path);
if (itemType === "dependencies") {
// Empty string is valid for workspace deps (means "no deps") — only skip undefined
if (content === undefined) continue;
const info = workspaceDependenciesPathToLanguageAndFilename(path);
if (info) {
const hash = await generateHash(content);
workspaceDeps.push({ path, language: info.language as ScriptLang, name: info.name, hash });
}
} else if (itemType === "script") {
if (!content) continue;
const hash = await generateHash(content);
scriptHashes[path] = hash;
}
// Skip inline_script, flow, app — they don't need temp storage uploads
}
if (Object.keys(scriptHashes).length === 0 && workspaceDeps.length === 0) return;
// Single batch query: find which scripts/deps differ from deployed versions
const mismatched = await wmill.diffRawScriptsWithDeployed({
workspace: workspace.workspaceId,
requestBody: {
scripts: scriptHashes,
workspace_deps: workspaceDeps,
},
});
// Upload only mismatched scripts to temp storage
for (const path of mismatched) {
const content = tree.getContent(path);
const itemType = tree.getItemType(path);
if (itemType === "dependencies") {
// Workspace deps don't need temp storage — just mark as mismatched.
// Empty string is valid (means the dep file was emptied locally).
if (content !== undefined) {
tree.setContentHash(path, "mismatched");
}
} else if (content) {
const hash = await wmill.storeRawScriptTemp({
workspace: workspace.workspaceId,
requestBody: content,
});
tree.setContentHash(path, hash);
}
}
}
export type ItemType = "script" | "inline_script" | "flow" | "app" | "dependencies";
interface DependencyNode {
content: string;
stalenessHash: string; // Hash for staleness detection (includes deps, content, metadata)
contentHash?: string; // Hash for temp storage lookup (content only)
language: ScriptLanguage;
metadata: string;
imports: Set<string>;
importedBy: Set<string>;
staleReason?: string;
// Item metadata for generate-metadata command
itemType: ItemType;
folder: string; // Folder path (for flows/apps) or remote path (for scripts)
originalPath: string; // Original path passed to handler (with extension for scripts)
isRawApp?: boolean; // Only set for apps
isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale)
}
export class DoubleLinkedDependencyTree {
private nodes: Map<string, DependencyNode> = new Map();
private workspaceDeps: Record<string, string> = {};
setWorkspaceDeps(deps: Record<string, string>): void {
this.workspaceDeps = deps;
}
async addNode(
path: string,
content: string,
language: ScriptLanguage,
metadata: string,
imports: string[],
itemType: ItemType,
folder: string,
originalPath: string,
isDirectlyStale: boolean,
isRawApp?: boolean
): Promise<void> {
const hasWorkspaceDeps = itemType === "script" || itemType === "inline_script";
const filteredDeps = hasWorkspaceDeps
? filterWorkspaceDependencies(this.workspaceDeps, content, language)
: {};
const stalenessHash = await generateScriptHash({}, content, metadata);
if (!this.nodes.has(path)) {
this.nodes.set(path, {
content: "", stalenessHash: "", language: "deno", metadata: "",
imports: new Set(), importedBy: new Set(),
itemType: "script", folder: "", originalPath: "", isDirectlyStale: false,
});
}
const node = this.nodes.get(path)!;
node.content = content;
node.stalenessHash = stalenessHash;
node.language = language;
node.metadata = metadata;
node.itemType = itemType;
node.folder = folder;
node.originalPath = originalPath;
node.isDirectlyStale = isDirectlyStale;
node.isRawApp = isRawApp;
// Create nodes for referenced workspace deps with content and language.
const filteredDepsPaths = Object.keys(filteredDeps);
for (const depsPath of filteredDepsPaths) {
if (!this.nodes.has(depsPath)) {
const depsInfo = workspaceDependenciesPathToLanguageAndFilename(depsPath);
const contentHash = await generateHash(filteredDeps[depsPath] + depsPath);
const isUpToDate = await checkifMetadataUptodate(depsPath, contentHash, undefined);
this.nodes.set(depsPath, {
content: filteredDeps[depsPath],
stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "",
imports: new Set(), importedBy: new Set(),
itemType: "dependencies", folder: "", originalPath: depsPath,
isDirectlyStale: !isUpToDate,
});
}
}
const allImports = [...imports, ...filteredDepsPaths];
for (const importPath of allImports) {
node.imports.add(importPath);
if (!this.nodes.has(importPath)) {
this.nodes.set(importPath, {
content: "", stalenessHash: "", language: "deno", metadata: "",
imports: new Set(), importedBy: new Set(),
itemType: "script", folder: "", originalPath: "", isDirectlyStale: false,
});
}
this.nodes.get(importPath)!.importedBy.add(path);
}
}
getContent(path: string): string | undefined {
return this.nodes.get(path)?.content;
}
getStalenessHash(path: string): string | undefined {
return this.nodes.get(path)?.stalenessHash;
}
getContentHash(path: string): string | undefined {
return this.nodes.get(path)?.contentHash;
}
setContentHash(path: string, hash: string): void {
const node = this.nodes.get(path);
if (node) {
node.contentHash = hash;
}
}
getLanguage(path: string): ScriptLanguage | undefined {
return this.nodes.get(path)?.language;
}
getMetadata(path: string): string | undefined {
return this.nodes.get(path)?.metadata;
}
getStaleReason(path: string): string | undefined {
return this.nodes.get(path)?.staleReason;
}
getItemType(path: string): ItemType | undefined {
return this.nodes.get(path)?.itemType;
}
getFolder(path: string): string | undefined {
return this.nodes.get(path)?.folder;
}
getIsRawApp(path: string): boolean | undefined {
return this.nodes.get(path)?.isRawApp;
}
getIsDirectlyStale(path: string): boolean {
return this.nodes.get(path)?.isDirectlyStale ?? false;
}
getOriginalPath(path: string): string | undefined {
return this.nodes.get(path)?.originalPath;
}
getImports(path: string): Set<string> | undefined {
return this.nodes.get(path)?.imports;
}
/**
* Returns true if this node has been marked stale (directly or transitively).
*/
isStale(path: string): boolean {
return this.nodes.get(path)?.staleReason !== undefined;
}
/**
* Mutates the tree by removing all nodes that are not stale.
* Uses BFS on reverse graph (importedBy) to find all stale scripts.
* Starts from nodes with isDirectlyStale=true.
*/
propagateStaleness(): void {
// Collect directly stale nodes
const directlyStale = new Set<string>();
for (const [path, node] of this.nodes.entries()) {
if (node.isDirectlyStale) {
directlyStale.add(path);
node.staleReason = "content changed";
}
}
const allStale = new Set(directlyStale);
const queue = [...directlyStale];
const visited = new Set<string>();
while (queue.length > 0) {
const scriptPath = queue.shift()!;
if (visited.has(scriptPath)) continue;
visited.add(scriptPath);
const node = this.nodes.get(scriptPath);
if (!node) continue;
for (const importer of node.importedBy) {
if (!allStale.has(importer)) {
allStale.add(importer);
queue.push(importer);
// Set reason for transitively stale scripts
const importerNode = this.nodes.get(importer);
if (importerNode) importerNode.staleReason = `depends on ${scriptPath}`;
}
}
}
}
/**
* Walks all transitive imports for a node, calling the callback for each.
* Callback may return true to stop traversing that branch.
*/
traverseTransitive(scriptPath: string, callback: (importPath: string, node: DependencyNode) => boolean | void): void {
const queue = [scriptPath];
const visited = new Set<string>();
while (queue.length > 0) {
const current = queue.shift()!;
if (visited.has(current)) continue;
visited.add(current);
const node = this.nodes.get(current);
if (!node) continue;
for (const importPath of node.imports) {
const importNode = this.nodes.get(importPath);
if (importNode) {
const stop = callback(importPath, importNode);
if (!stop) {
queue.push(importPath);
}
}
}
}
}
allPaths(): IterableIterator<string> {
return this.nodes.keys();
}
/**
* Returns paths of all stale nodes (those with a staleReason).
*/
*stalePaths(): IterableIterator<string> {
for (const [path, node] of this.nodes.entries()) {
if (node.staleReason) {
yield path;
}
}
}
has(path: string): boolean {
return this.nodes.has(path);
}
/**
* Returns workspace deps that were uploaded as mismatched with remote.
* These need to be passed as raw_workspace_dependencies in job args
* so the backend uses local content instead of deployed.
*/
getMismatchedWorkspaceDeps(): Record<string, string> {
const result: Record<string, string> = {};
for (const [path, node] of this.nodes.entries()) {
if (node.itemType === "dependencies" && node.contentHash && node.content !== undefined) {
result[path] = node.content;
}
}
return result;
}
/**
* Returns path → contentHash for all transitive imports that have been uploaded.
* Must be called after uploadScripts() has populated contentHash values.
*/
getTempScriptRefs(scriptPath: string): Record<string, string> {
const result: Record<string, string> = {};
this.traverseTransitive(scriptPath, (_path, node) => {
if (node.contentHash) {
result[_path] = node.contentHash;
}
});
return result;
}
/**
* Persist workspace dep hashes to wmill-lock.yaml so getRawWorkspaceDependencies
* considers them up-to-date on the next run.
*/
async persistDepsHashes(depsPaths: string[]): Promise<void> {
for (const path of depsPaths) {
const node = this.nodes.get(path);
if (node?.itemType === "dependencies" && node.content !== undefined) {
const hash = await generateHash(node.content + path);
await updateMetadataGlobalLock(path, hash);
}
}
}
get size(): number {
return this.nodes.size;
}
}
+155
View File
@@ -0,0 +1,155 @@
import { execFileSync } from "node:child_process";
import { readFile, stat } from "node:fs/promises";
import type { SyncCodebase } from "./codebase.ts";
import { parseMetadataFileIfExists } from "./metadata.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { findCodebase } from "../commands/sync/sync.ts";
import type { LocalScriptInfo } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts";
import type { RawScript } from "../../../gen/types.gen.ts";
export class UnsupportedLocalPathScriptPreviewError extends Error {
constructor(message: string) {
super(message);
this.name = "UnsupportedLocalPathScriptPreviewError";
}
}
async function readOptionalLock(scriptPath: string): Promise<string | undefined> {
try {
return await readFile(scriptPath + ".script.lock", "utf-8");
} catch {
return undefined;
}
}
function normalizeOptionalLock(lock: string | undefined): string | undefined {
return typeof lock === "string" && lock.trim() === "" ? undefined : lock;
}
async function bundleSingleFileCodebaseScript(
filePath: string,
codebase: SyncCodebase
): Promise<string> {
if (codebase.customBundler) {
// Pass the script path as a positional shell argument so existing shell-based
// custom bundlers still work without interpolating the path into the command.
return execFileSync(
"sh",
["-lc", `${codebase.customBundler} "$1"`, "sh", filePath],
{
maxBuffer: 1024 * 1024 * 50,
}
).toString();
}
const esbuild = await import("esbuild");
const out = await esbuild.build({
entryPoints: [filePath],
// Inline rawscripts are executed through the standard module wrapper,
// so the bundle must expose `main` as an ESM export.
format: "esm",
bundle: true,
write: false,
external: codebase.external,
inject: codebase.inject,
define: codebase.define,
loader: codebase.loader ?? { ".node": "file" },
outdir: "/",
platform: "node",
packages: "bundle",
target: "esnext",
banner: codebase.banner,
});
if (out.outputFiles.length === 0) {
throw new Error(`No output files found for ${filePath}`);
}
if (out.outputFiles.length > 1) {
throw new UnsupportedLocalPathScriptPreviewError(
`Local PathScript ${filePath} requires a multi-file bundle, which flow preview/dev cannot inline yet`
);
}
if (Array.isArray(codebase.assets) && codebase.assets.length > 0) {
throw new UnsupportedLocalPathScriptPreviewError(
`Local PathScript ${filePath} requires codebase assets, which flow preview/dev cannot inline yet`
);
}
return out.outputFiles[0].text;
}
export function createPreviewLocalScriptReader(opts: {
exts: string[];
defaultTs?: "bun" | "deno";
codebases: SyncCodebase[];
}): (scriptPath: string) => Promise<LocalScriptInfo | undefined> {
return async (scriptPath) => {
const localScript = await resolvePreviewLocalScriptState(scriptPath, opts);
if (!localScript) {
return undefined;
}
const content = localScript.codebase
? await bundleSingleFileCodebaseScript(localScript.filePath, localScript.codebase)
: localScript.content;
return {
content,
language: localScript.language,
lock: localScript.lock,
tag: localScript.tag,
};
};
}
export type PreviewLocalScriptState = {
filePath: string;
content: string;
language: RawScript["language"];
lock?: string;
tag?: string;
codebase?: SyncCodebase;
codebaseDigest?: string;
};
export async function resolvePreviewLocalScriptState(
scriptPath: string,
opts: {
exts: string[];
defaultTs?: "bun" | "deno";
codebases: SyncCodebase[];
}
): Promise<PreviewLocalScriptState | undefined> {
for (const ext of opts.exts) {
const filePath = scriptPath + ext;
let fileStat;
try {
fileStat = await stat(filePath);
} catch {
continue;
}
if (!fileStat.isFile()) continue;
const language = inferContentTypeFromFilePath(filePath, opts.defaultTs);
const metadata = await parseMetadataFileIfExists(scriptPath);
const rawLock = metadata?.payload?.lock ?? (await readOptionalLock(scriptPath));
const codebase =
language === "bun" ? findCodebase(filePath, opts.codebases) : undefined;
return {
filePath,
content: await readFile(filePath, "utf-8"),
language,
lock: normalizeOptionalLock(rawLock),
tag: metadata?.payload?.tag,
codebase,
codebaseDigest: codebase
? await codebase.getDigest(
Array.isArray(codebase.assets) && codebase.assets.length > 0
)
: undefined,
};
}
return undefined;
}
+425 -107
View File
@@ -5,7 +5,8 @@ import * as log from "../core/log.ts";
import { stringify as yamlStringify } from "yaml";
import { yamlParseFile } from "./yaml.ts";
import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises";
import { readFileSync } from "node:fs";
import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
import * as path from "node:path";
import { createRequire } from "node:module";
import {
ScriptMetadata,
@@ -15,19 +16,23 @@ import { Workspace } from "../commands/workspace/workspace.ts";
import {
ScriptLanguage,
workspaceDependenciesLanguages,
languageNeedsLock,
} from "./script_common.ts";
import { inferContentTypeFromFilePath } from "./script_common.ts";
import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts";
import { findCodebase, yamlOptions } from "../commands/sync/sync.ts";
import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts";
import { SyncCodebase } from "./codebase.ts";
import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts";
import { getIsWin } from "./utils.ts";
import { extractRelativeImports } from "./relative_imports.ts";
import { DoubleLinkedDependencyTree } from "./dependency_tree.ts";
const _require = createRequire(import.meta.url);
const _parserCache = new Map<string, Promise<any>>();
function loadParser(pkgName: string): Promise<any> {
export function loadParser(pkgName: string): Promise<any> {
let p = _parserCache.get(pkgName);
if (!p) {
p = (async () => {
@@ -35,7 +40,7 @@ function loadParser(pkgName: string): Promise<any> {
const wasmPath = _require.resolve(
`${pkgName}/windmill_parser_wasm_bg.wasm`
);
await mod.default(readFileSync(wasmPath));
await mod.default({ module_or_path: readFileSync(wasmPath) });
return mod;
})();
_parserCache.set(pkgName, p);
@@ -50,27 +55,28 @@ export class LockfileGenerationError extends Error {
}
}
export async function generateAllMetadata() {}
export async function getRawWorkspaceDependencies(): Promise<Record<string, string>> {
export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise<Record<string, string>> {
const rawWorkspaceDeps: Record<string, string> = {};
try {
const entries = await readdir("dependencies", { withFileTypes: true });
for (const entry of entries) {
if (entry.isDirectory()) continue;
const filePath = `dependencies/${entry.name}`;
const content = await readFile(filePath, "utf-8");
// Find matching language
for (const lang of workspaceDependenciesLanguages) {
if (entry.name.endsWith(lang.filename)) {
// Check if out of sync
const contentHash = await generateHash(content + filePath);
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
if (!isUpToDate) {
if (legacyBehaviour) {
const contentHash = await generateHash(content + filePath);
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
if (!isUpToDate) {
rawWorkspaceDeps[filePath] = content;
}
} else {
rawWorkspaceDeps[filePath] = content;
}
break;
@@ -184,15 +190,22 @@ export async function generateScriptMetadataInternal(
noStaleMessage: boolean,
rawWorkspaceDependencies: Record<string, string>,
codebases: SyncCodebase[],
justUpdateMetadataLock?: boolean
justUpdateMetadataLock?: boolean,
legacyBehaviour?: boolean,
tree?: DoubleLinkedDependencyTree
): Promise<string | undefined> {
const remotePath = scriptPath
.substring(0, scriptPath.indexOf("."))
.replaceAll(SEP, "/");
// Detect folder layout: my_script__mod/script.ts
const isFolderLayout = isModuleEntryPoint(scriptPath);
// remotePath is the Windmill API path (e.g., "u/admin/my_script")
const remotePath = isFolderLayout
? getScriptBasePathFromModulePath(scriptPath)!.replaceAll(SEP, "/")
: scriptPath.substring(0, scriptPath.indexOf(".")).replaceAll(SEP, "/");
const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
// For folder layout, parseMetadataFile is called with remotePath which
// will find __mod/script.yaml via the folder layout fallback
const metadataWithType = await parseMetadataFile(
remotePath,
undefined,
@@ -208,22 +221,76 @@ export async function generateScriptMetadataInternal(
language
);
// Compute the module folder path early so we can include module hashes in stale check
const moduleFolderPath = isFolderLayout
? path.dirname(scriptPath)
: scriptPath.substring(0, scriptPath.indexOf(".")) + getModuleFolderSuffix();
// Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically
let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent);
const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory();
if (await checkifMetadataUptodate(remotePath, hash, undefined)) {
if (!noStaleMessage) {
log.info(
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
return `${remotePath} (${language})`;
// In non-legacy mode, workspace deps are tracked via the tree — exclude from hash
const depsForHash = (!legacyBehaviour && tree) ? {} : filteredRawWorkspaceDependencies;
let hash = await generateScriptHash(depsForHash, scriptContent, metadataContent);
// Compute per-module hashes for stale detection (like flow inline scripts)
let moduleHashes: Record<string, string> = {};
if (hasModules) {
moduleHashes = await computeModuleHashes(
moduleFolderPath, opts.defaultTs, (!legacyBehaviour && tree) ? {} : rawWorkspaceDependencies, isFolderLayout
);
}
const hasModuleHashes = Object.keys(moduleHashes).length > 0;
// If modules exist, combine main script hash + module hashes into a meta-hash
let checkHash = hash;
let checkSubpath: string | undefined;
if (hasModuleHashes) {
const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b));
checkHash = await generateHash(hash + JSON.stringify(sortedEntries));
checkSubpath = SCRIPT_TOP_HASH;
}
if (!justUpdateMetadataLock) {
const conf = await readLockfile();
// Use checkHash (includes module hashes) so module changes are detected as stale
const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath));
// New behaviour: tree-based dependency tracking
if (!legacyBehaviour && tree) {
if (dryRun) {
// First pass: populate tree with script and its imports
const imports = await extractRelativeImports(scriptContent, remotePath, language);
await tree.addNode(remotePath, scriptContent, language, metadataContent, imports, "script", remotePath, scriptPath, isDirectlyStale);
return;
}
// Second pass: proceed to generate (caller verified this script is stale via tree)
} else {
// Legacy behaviour: use existing staleness check
if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) {
if (!noStaleMessage) {
log.info(
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
let detail = `${remotePath} (${language})`;
if (hasModuleHashes) {
const changed: string[] = [];
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) {
changed.push(modulePath);
}
}
if (changed.length > 0) {
detail += ` [changed modules: ${changed.join(", ")}]`;
}
}
return detail;
}
}
if (!justUpdateMetadataLock && !noStaleMessage) {
log.info(colors.gray(`Generating metadata for ${scriptPath}`));
}
@@ -245,36 +312,97 @@ export async function generateScriptMetadataInternal(
const hasCodebase = findCodebase(scriptPath, codebases) != undefined;
if (!hasCodebase) {
const tempScriptRefs = tree?.getTempScriptRefs(remotePath);
const lockPathOverride = isFolderLayout
? path.dirname(scriptPath) + "/script.lock"
: undefined;
await updateScriptLock(
workspace,
scriptContent,
language,
remotePath,
metadataParsedContent,
filteredRawWorkspaceDependencies
filteredRawWorkspaceDependencies,
tempScriptRefs,
lockPathOverride,
);
} else {
metadataParsedContent.lock = "";
}
// Generate locks for modules in __mod/ folder
if (hasModules) {
// Identify which modules changed by comparing per-module hashes
let changedModules: string[] | undefined;
if (hasModuleHashes) {
changedModules = [];
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) {
changedModules.push(modulePath);
}
}
if (changedModules.length === 0) {
changedModules = undefined; // no modules changed, skip lock regeneration
}
}
await updateModuleLocks(
workspace, moduleFolderPath, "", remotePath,
rawWorkspaceDependencies, opts.defaultTs, changedModules,
);
}
} else {
metadataParsedContent.lock =
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
if (isFolderLayout) {
metadataParsedContent.lock =
"!inline " + remotePath.replaceAll(SEP, "/") + getModuleFolderSuffix() + "/script.lock";
} else {
metadataParsedContent.lock =
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
}
}
let metaPath = remotePath + ".script.yaml";
let newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
if (metadataWithType.isJson) {
metaPath = remotePath + ".script.json";
newMetadataContent = JSON.stringify(metadataParsedContent);
// Write metadata back to the correct path
let metaPath: string;
let newMetadataContent: string;
if (isFolderLayout) {
if (metadataWithType.isJson) {
metaPath = path.dirname(scriptPath) + "/script.json";
newMetadataContent = JSON.stringify(metadataParsedContent);
} else {
metaPath = path.dirname(scriptPath) + "/script.yaml";
newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
}
} else {
if (metadataWithType.isJson) {
metaPath = remotePath + ".script.json";
newMetadataContent = JSON.stringify(metadataParsedContent);
} else {
metaPath = remotePath + ".script.yaml";
newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
}
}
const metadataContentUsedForHash = newMetadataContent;
hash = await generateScriptHash(
filteredRawWorkspaceDependencies,
depsForHash,
scriptContent,
metadataContentUsedForHash
);
await updateMetadataGlobalLock(remotePath, hash);
// Store hashes in wmill-lock.yaml
if (hasModuleHashes) {
// Use per-module hash tracking (like flow inline scripts)
const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b));
const metaHash = await generateHash(hash + JSON.stringify(sortedEntries));
await clearGlobalLock(remotePath);
await updateMetadataGlobalLock(remotePath, metaHash, SCRIPT_TOP_HASH);
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
await updateMetadataGlobalLock(remotePath, moduleHash, modulePath);
}
} else {
await updateMetadataGlobalLock(remotePath, hash);
}
if (!justUpdateMetadataLock) {
await writeFile(metaPath, newMetadataContent, "utf-8");
}
@@ -300,11 +428,9 @@ export async function updateScriptSchema(
} else {
delete metadataContent.has_preprocessor;
}
if (result.no_main_func) {
metadataContent.no_main_func = result.no_main_func;
} else {
delete metadataContent.no_main_func;
}
// auto_kind is intentionally not written to metadata — it is auto-detected
// by the parser at deploy time from script content.
delete metadataContent.auto_kind;
}
// ---------------------------------------------------------------------------
@@ -329,6 +455,7 @@ const LANG_ANNOTATION_CONFIG: Partial<
nativets: { comment: "//", keyword: "package_json" },
go: { comment: "//", keyword: "go_mod" },
php: { comment: "//", keyword: "composer_json" },
powershell: { comment: "#", keyword: "modules_json" },
};
export function extractWorkspaceDepsAnnotation(
@@ -410,6 +537,7 @@ export async function computeLockCacheKey(
scriptContent: string,
language: ScriptLanguage,
rawWorkspaceDependencies: Record<string, string>,
tempScriptRefs?: Record<string, string>
): Promise<string> {
const annotation = extractWorkspaceDepsAnnotation(scriptContent, language);
const annotationStr = annotation
@@ -417,7 +545,10 @@ export async function computeLockCacheKey(
: "none";
const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort();
const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";");
return await generateHash(`${language}|${annotationStr}|${depsStr}`);
const tempRefsStr = tempScriptRefs
? Object.keys(tempScriptRefs).sort().map((k) => `${k}=${tempScriptRefs[k]}`).join(";")
: "";
return await generateHash(`${language}|${annotationStr}|${depsStr}|${tempRefsStr}`);
}
const lockCache = new Map<string, string>();
@@ -432,13 +563,15 @@ async function fetchScriptLock(
language: ScriptLanguage,
remotePath: string,
rawWorkspaceDependencies: Record<string, string>,
tempScriptRefs?: Record<string, string>
): Promise<string> {
const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0;
const cacheKey = hasRawDeps
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies)
const hasTempRefs = tempScriptRefs && Object.keys(tempScriptRefs).length > 0;
const cacheKey = (hasRawDeps || hasTempRefs)
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies, tempScriptRefs)
: undefined;
if (cacheKey && lockCache.has(cacheKey)) {
log.info(`Using cached lockfile for ${remotePath}`);
log.debug(`Using cached lockfile for ${remotePath}`);
return lockCache.get(cacheKey)!;
}
@@ -463,6 +596,8 @@ async function fetchScriptLock(
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
? rawWorkspaceDependencies : null,
entrypoint: remotePath,
temp_script_refs: tempScriptRefs && Object.keys(tempScriptRefs).length > 0
? tempScriptRefs : null,
}),
}
);
@@ -502,11 +637,14 @@ async function updateScriptLock(
language: ScriptLanguage,
remotePath: string,
metadataContent: Record<string, any>,
rawWorkspaceDependencies: Record<string, string>
rawWorkspaceDependencies: Record<string, string>,
tempScriptRefs?: Record<string, string>,
lockPathOverride?: string,
): Promise<void> {
if (
!(
workspaceDependenciesLanguages.some((l) => l.language == language) ||
(workspaceDependenciesLanguages.some((l) => l.language == language) &&
language !== "powershell") ||
language == "deno" ||
language == "rust" ||
language == "ansible"
@@ -518,7 +656,7 @@ async function updateScriptLock(
if (Object.keys(rawWorkspaceDependencies).length > 0) {
const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', ');
log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
log.debug(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
}
const lock = await fetchScriptLock(
@@ -527,9 +665,10 @@ async function updateScriptLock(
language,
remotePath,
rawWorkspaceDependencies,
tempScriptRefs
);
const lockPath = remotePath + ".script.lock";
const lockPath = lockPathOverride ?? remotePath + ".script.lock";
if (lock != "") {
await writeFile(lockPath, lock, "utf-8");
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
@@ -545,6 +684,82 @@ async function updateScriptLock(
}
}
/**
* Generate locks for all module files in a __mod/ directory.
* Recursively walks the directory and generates a lock for each module
* whose language requires one.
*/
async function updateModuleLocks(
workspace: Workspace,
dirPath: string,
relPrefix: string,
scriptRemotePath: string,
rawWorkspaceDependencies: Record<string, string>,
defaultTs: "bun" | "deno" | undefined,
changedModules?: string[],
): Promise<void> {
const entries = readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
if (entry.isDirectory()) {
await updateModuleLocks(workspace, fullPath, relPath, scriptRemotePath, rawWorkspaceDependencies, defaultTs, changedModules);
} else if (entry.isFile()
&& !entry.name.endsWith(".lock")
// In folder layout, skip entry point files (script.{ext}, script.yaml, script.json, script.lock)
&& !(relPrefix === "" && entry.name.startsWith("script."))
) {
let modLanguage: ScriptLanguage;
try {
modLanguage = inferContentTypeFromFilePath(entry.name, defaultTs);
} catch {
continue; // skip files with unrecognized extensions
}
if (!languageNeedsLock(modLanguage)) continue;
// Skip unchanged modules when per-module hash tracking is active
if (changedModules) {
const normalizedRelPath = normalizeLockPath(relPath);
if (!changedModules.includes(normalizedRelPath)) continue;
}
const moduleContent = readFileSync(fullPath, "utf-8");
const moduleRemotePath = scriptRemotePath + "/" + relPath;
log.debug(`Generating lock for module ${relPath}`);
try {
const lock = await fetchScriptLock(
workspace,
moduleContent,
modLanguage,
moduleRemotePath,
rawWorkspaceDependencies,
);
const baseName = entry.name.replace(/\.[^.]+$/, '');
const lockPath = path.join(dirPath, baseName + ".lock");
if (lock != "") {
writeFileSync(lockPath, lock, "utf-8");
} else {
try {
if (existsSync(lockPath)) {
const { rm: rmAsync } = await import("node:fs/promises");
await rmAsync(lockPath);
}
} catch {
// ignore
}
}
} catch (e) {
log.info(colors.yellow(`Failed to generate lock for module ${relPath}: ${e}`));
}
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////
// below functions copied from Windmill's FE inferArgs function. TODO: refactor //
////////////////////////////////////////////////////////////////////////////////////////////
@@ -556,7 +771,7 @@ export async function inferSchema(
): Promise<{
schema: any;
has_preprocessor: boolean | undefined;
no_main_func: boolean | undefined;
auto_kind: string | undefined;
}> {
let inferedSchema: any;
if (language === "python3") {
@@ -666,7 +881,7 @@ export async function inferSchema(
return {
schema: defaultScriptMetadata().schema,
has_preprocessor: false,
no_main_func: false,
auto_kind: undefined,
};
}
@@ -691,6 +906,11 @@ export async function inferSchema(
argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]);
// For T | T[] detection for debouncing arg accumulation
if ((arg as any).otyp && (arg as any).otyp.includes('[') && (arg as any).otyp.includes('|')) {
currentSchema.properties[arg.name].originalType = (arg as any).otyp
}
currentSchema.properties[arg.name].default = arg.default;
if (!arg.has_default && !currentSchema.required.includes(arg.name)) {
@@ -701,7 +921,7 @@ export async function inferSchema(
return {
schema: currentSchema,
has_preprocessor: inferedSchema.has_preprocessor,
no_main_func: inferedSchema.no_main_func,
auto_kind: inferedSchema.auto_kind,
};
}
@@ -737,6 +957,38 @@ export function replaceLock(o?: { lock?: string | string[] }) {
}
}
}
export async function parseMetadataFileIfExists(
scriptPath: string
): Promise<{ isJson: boolean; payload: any; path: string } | undefined> {
let metadataFilePath = scriptPath + ".script.json";
try {
await stat(metadataFilePath);
const payload = JSON.parse(await readFile(metadataFilePath, "utf-8"));
replaceLock(payload);
return {
path: metadataFilePath,
payload,
isJson: true,
};
} catch {
try {
metadataFilePath = scriptPath + ".script.yaml";
await stat(metadataFilePath);
const payload: any = await yamlParseFile(metadataFilePath);
replaceLock(payload);
return {
path: metadataFilePath,
payload,
isJson: false,
};
} catch {
return undefined;
}
}
}
export async function parseMetadataFile(
scriptPath: string,
generateMetadataIfMissing:
@@ -770,62 +1022,87 @@ export async function parseMetadataFile(
isJson: false,
};
} catch {
// no metadata file at all. Create it
log.info(
(await blueColor())(
`Creating script metadata file for ${metadataFilePath}`
)
);
metadataFilePath = scriptPath + ".script.yaml";
let scriptInitialMetadata = defaultScriptMetadata();
const lockPath = scriptPath + ".script.lock";
scriptInitialMetadata.lock = "!inline " + lockPath;
const scriptInitialMetadataYaml = yamlStringify(
scriptInitialMetadata as Record<string, any>,
yamlOptions
);
await writeFile(metadataFilePath, scriptInitialMetadataYaml, { flag: "wx", encoding: "utf-8" });
await writeFile(lockPath, "", { flag: "wx", encoding: "utf-8" });
if (generateMetadataIfMissing) {
log.info(
(await blueColor())(
`Generating lockfile and schema for ${metadataFilePath}`
)
);
// Try folder layout: {scriptPath}__mod/script.yaml or .json
const moduleFolderMeta = scriptPath + getModuleFolderSuffix();
try {
metadataFilePath = moduleFolderMeta + "/script.json";
await stat(metadataFilePath);
return {
path: metadataFilePath,
payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
isJson: true,
};
} catch {
try {
await generateScriptMetadataInternal(
generateMetadataIfMissing.path,
generateMetadataIfMissing.workspaceRemote,
generateMetadataIfMissing,
false,
false,
generateMetadataIfMissing.rawWorkspaceDependencies,
generateMetadataIfMissing.codebases,
false
);
scriptInitialMetadata = (await yamlParseFile(
metadataFilePath
)) as ScriptMetadata;
if (!generateMetadataIfMissing.schemaOnly) {
replaceLock(scriptInitialMetadata);
}
} catch (e) {
log.info(
colors.yellow(
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
)
);
metadataFilePath = moduleFolderMeta + "/script.yaml";
await stat(metadataFilePath);
const payload: any = await yamlParseFile(metadataFilePath);
replaceLock(payload);
return {
path: metadataFilePath,
payload,
isJson: false,
};
} catch {
// fall through to create metadata
}
}
return {
path: metadataFilePath,
payload: scriptInitialMetadata,
isJson: false,
};
}
}
// no metadata file at all. Create it
log.info(
(await blueColor())(
`Creating script metadata file for ${metadataFilePath}`
)
);
metadataFilePath = scriptPath + ".script.yaml";
let scriptInitialMetadata = defaultScriptMetadata();
const lockPath = scriptPath + ".script.lock";
scriptInitialMetadata.lock = "!inline " + lockPath;
const scriptInitialMetadataYaml = yamlStringify(
scriptInitialMetadata as Record<string, any>,
yamlOptions
);
await writeFile(metadataFilePath, scriptInitialMetadataYaml, { flag: "wx", encoding: "utf-8" });
await writeFile(lockPath, "", { flag: "wx", encoding: "utf-8" });
if (generateMetadataIfMissing) {
log.info(
(await blueColor())(
`Generating lockfile and schema for ${metadataFilePath}`
)
);
try {
await generateScriptMetadataInternal(
generateMetadataIfMissing.path,
generateMetadataIfMissing.workspaceRemote,
generateMetadataIfMissing,
false,
false,
generateMetadataIfMissing.rawWorkspaceDependencies,
generateMetadataIfMissing.codebases,
false
);
scriptInitialMetadata = (await yamlParseFile(
metadataFilePath
)) as ScriptMetadata;
if (!generateMetadataIfMissing.schemaOnly) {
replaceLock(scriptInitialMetadata);
}
} catch (e) {
log.info(
colors.yellow(
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
)
);
}
}
return {
path: metadataFilePath,
payload: scriptInitialMetadata,
isJson: false,
};
}
interface Lock {
@@ -834,6 +1111,7 @@ interface Lock {
}
const WMILL_LOCKFILE = "wmill-lock.yaml";
const SCRIPT_TOP_HASH = "__script_hash";
/**
* Normalizes a path to use Linux separators (forward slashes).
@@ -902,6 +1180,46 @@ export async function generateScriptHash(
);
}
async function computeModuleHashes(
moduleFolderPath: string,
defaultTs: "bun" | "deno" | undefined,
rawWorkspaceDependencies: Record<string, string>,
isFolderLayout: boolean,
): Promise<Record<string, string>> {
const hashes: Record<string, string> = {};
async function readDir(dirPath: string, relPrefix: string) {
const entries = readdirSync(dirPath, { withFileTypes: true });
for (const entry of entries) {
const fullPath = path.join(dirPath, entry.name);
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
const isTopLevel = relPrefix === "";
if (entry.isDirectory()) {
await readDir(fullPath, relPath);
} else if (
entry.isFile() &&
!entry.name.endsWith(".lock") &&
!(isFolderLayout && isTopLevel && entry.name.startsWith("script."))
) {
try {
inferContentTypeFromFilePath(entry.name, defaultTs);
} catch {
continue;
}
const content = readFileSync(fullPath, "utf-8");
const normalizedPath = normalizeLockPath(relPath);
hashes[normalizedPath] = await generateHash(
content + JSON.stringify(rawWorkspaceDependencies)
);
}
}
}
await readDir(moduleFolderPath, "");
return hashes;
}
export async function clearGlobalLock(path: string): Promise<void> {
const conf = await readLockfile();
if (!conf?.locks) {
+39
View File
@@ -0,0 +1,39 @@
/**
* Relative Imports Utilities for CLI
*
* Provides functions to parse relative imports from TypeScript/Python scripts using WASM.
*/
import { ScriptLanguage } from "./script_common.ts";
import { loadParser } from "./metadata.ts";
import * as log from "../core/log.ts";
/**
* Extract relative imports from script content based on language.
* Returns resolved absolute Windmill paths (e.g., "f/folder/helper").
*/
export async function extractRelativeImports(
code: string,
scriptPath: string,
language: ScriptLanguage
): Promise<string[]> {
try {
switch (language) {
case "bun":
case "nativets":
case "deno": {
const { parse_ts_relative_imports } = await loadParser("windmill-parser-wasm-ts");
return parse_ts_relative_imports(code, scriptPath);
}
case "python3": {
const { parse_py_relative_imports } = await loadParser("windmill-parser-wasm-py-imports");
return parse_py_relative_imports(code, scriptPath);
}
default:
return [];
}
} catch (e) {
log.warn(`Failed to parse relative imports for ${scriptPath}: ${e}. Dependency tracking for relative imports will be disabled.`);
return [];
}
}
+80
View File
@@ -189,6 +189,26 @@ export function isFolderResourcePath(p: string): boolean {
return isFlowPath(p) || isAppPath(p) || isRawAppPath(p);
}
/**
* Check if a path is inside a folder-based resource, checking BOTH dotted (.flow, .app, .raw_app)
* and non-dotted (__flow, __app, __raw_app) formats regardless of the global nonDottedPaths setting.
* Use this instead of isFolderResourcePath when the config may not yet be loaded or when
* you need to handle mixed-format workspaces (e.g. generate-metadata scanning all files).
*/
export function isFolderResourcePathAnyFormat(p: string): boolean {
const n = normalizeSep(p);
for (const suffixes of [DOTTED_SUFFIXES, NON_DOTTED_SUFFIXES]) {
if (
n.includes(suffixes.flow + "/") ||
n.includes(suffixes.app + "/") ||
n.includes(suffixes.raw_app + "/")
) {
return true;
}
}
return false;
}
/**
* Detect the resource type from a path, if any
*/
@@ -433,6 +453,66 @@ export function isRawAppFolderMetadataFile(p: string): boolean {
);
}
// ============================================================================
// Script Module Path Functions
// ============================================================================
/**
* The suffix used for script module folders.
* Unlike flows/apps, modules always use `__mod` (never dotted `.mod`)
* to avoid confusion with file extensions.
*/
const MODULE_SUFFIX = "__mod";
/**
* Get the module folder suffix (always "__mod")
*/
export function getModuleFolderSuffix(): string {
return MODULE_SUFFIX;
}
/**
* Check if a path is inside a script module folder.
* Matches patterns like: .../my_script__mod/...
*/
export function isScriptModulePath(p: string): boolean {
return normalizeSep(p).includes(MODULE_SUFFIX + "/");
}
/**
* Build the module folder path from a script's base path (without extension).
* e.g., "f/my_script" -> "f/my_script__mod"
*/
export function buildModuleFolderPath(scriptBasePath: string): string {
return scriptBasePath + MODULE_SUFFIX;
}
/**
* Check if a file inside a __mod/ folder is the main entry point (script.{ext}).
* Entry points are files named "script.*" directly under __mod/ (not in subdirs).
*/
export function isModuleEntryPoint(p: string): boolean {
const norm = normalizeSep(p);
const suffix = MODULE_SUFFIX + "/";
const idx = norm.indexOf(suffix);
if (idx === -1) return false;
const rest = norm.slice(idx + suffix.length);
return rest.startsWith("script.") && !rest.includes("/");
}
/**
* Extract the script base path from a module folder entry.
* e.g., "u/admin/my_script__mod/script.ts" -> "u/admin/my_script"
* e.g., "u/admin/my_script__mod/helper.ts" -> "u/admin/my_script"
*/
export function getScriptBasePathFromModulePath(p: string): string | undefined {
const norm = normalizeSep(p);
const suffix = MODULE_SUFFIX + "/";
const idx = norm.indexOf(suffix);
if (idx === -1) return undefined;
return norm.slice(0, idx);
}
// ============================================================================
// Sync-related Path Functions
// ============================================================================
+5 -2
View File
@@ -30,13 +30,15 @@ export type WorkspaceDependenciesLanguage =
| { language: "bun", filename /** (raw requirements filename) */: "package.json" }
| { language: "python3", filename: "requirements.in" }
| { language: "php", filename: "composer.json" }
| { language: "go", filename: "go.mod" };
| { language: "go", filename: "go.mod" }
| { language: "powershell", filename: "modules.json" };
export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
{ language: "bun", filename: "package.json" },
{ language: "python3", filename: "requirements.in" },
{ language: "php", filename: "composer.json" },
{ language: "go", filename: "go.mod" },
{ language: "powershell", filename: "modules.json" },
] as const;
/**
@@ -45,7 +47,8 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
*/
export function languageNeedsLock(language: ScriptLanguage | string): boolean {
return (
workspaceDependenciesLanguages.some((l) => l.language === language) ||
(workspaceDependenciesLanguages.some((l) => l.language === language) &&
language !== "powershell") ||
language === "deno" ||
language === "rust" ||
language === "ansible"
+27 -2
View File
@@ -63,11 +63,11 @@ export class CargoBackend {
// Determine default features based on environment
// CI mode: minimal features (zip only)
// Local mode with license key: full features (zip, private, enterprise, license)
// Local mode with license key: full features (zip, private, enterprise, license, python)
// Local mode without license key: zip only (EE features reject API calls without valid license)
const isCI = process.env["CI_MINIMAL_FEATURES"] === "true";
const hasLicenseKey = !!process.env["EE_LICENSE_KEY"];
const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license"] : ["zip"]);
const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license", "python"] : ["zip", "python"]);
// Parse additional features from environment variable
const envFeatures = process.env["TEST_FEATURES"]?.split(",").filter(f => f.trim()) || [];
@@ -328,6 +328,8 @@ export class CargoBackend {
SQLX_OFFLINE: "true",
// Disable embedding to speed up startup
DISABLE_EMBEDDING: "true",
// Skip worker version check for workspace deps (workers need time to report version)
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: "1",
// Create default admin user
CREATE_SUPERADMIN_IF_NOT_EXISTS: "1",
SUPERADMIN_EMAIL: this.config.username,
@@ -708,6 +710,7 @@ export class CargoBackend {
this.deleteAll("resources"),
this.deleteAll("variables"),
this.deleteAll("folders"),
this.deleteAllWorkspaceDeps(),
]);
console.log("Workspace reset complete");
@@ -735,6 +738,28 @@ export class CargoBackend {
// Ignore listing failures
}
}
private async deleteAllWorkspaceDeps(): Promise<void> {
try {
const listResponse = await this.apiRequest(`/api/w/${this.config.workspace}/workspace_dependencies/list`);
if (!listResponse.ok) return;
const items = await listResponse.json() as { language: string; name?: string }[];
for (const item of items) {
try {
const nameParam = item.name ? `?name=${encodeURIComponent(item.name)}` : "";
await this.apiRequest(
`/api/w/${this.config.workspace}/workspace_dependencies/delete/${item.language}${nameParam}`,
{ method: "POST" }
);
} catch {
// Ignore individual deletion failures
}
}
} catch {
// Ignore failures
}
}
}
// Global backend instance
+226
View File
@@ -0,0 +1,226 @@
/**
* Datatable settings sync tests
*
* Tests that datatable config is correctly synced via settings.yaml during pull/push operations.
*/
import { expect, test } from "bun:test";
import { writeFile, readFile } from "node:fs/promises";
import { parse, stringify } from "yaml";
import { withTestBackend } from "./test_backend.ts";
import { shouldSkipOnCI } from "./cargo_backend.ts";
import { addWorkspace } from "../workspace.ts";
test.skipIf(shouldSkipOnCI())("Datatable config: included in sync pull settings.yaml", async () => {
await withTestBackend(async (backend, tempDir) => {
const testWorkspace = {
remote: backend.baseUrl,
workspaceId: backend.workspace,
name: "datatable_pull_test",
token: backend.token
};
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
// Configure datatable on backend
const datatableConfig = {
datatables: {
main: {
database: {
resource_path: "u/test/test_db",
resource_type: "postgresql"
}
}
}
};
const configResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/workspaces/edit_datatable_config`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ settings: datatableConfig })
}
);
expect(configResp.ok).toBe(true);
// Create wmill.yaml with includeSettings
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes:
- "**"
includeSettings: true`, "utf-8");
// Pull settings
const result = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
expect(result.code).toEqual(0);
// Verify settings.yaml was created and contains datatable
const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8");
expect(settingsContent).toContain("datatable:");
expect(settingsContent).toContain("u/test/test_db");
expect(settingsContent).toContain("postgresql");
});
});
test.skipIf(shouldSkipOnCI())("Datatable config: pushed correctly from settings.yaml", async () => {
await withTestBackend(async (backend, tempDir) => {
const testWorkspace = {
remote: backend.baseUrl,
workspaceId: backend.workspace,
name: "datatable_push_test",
token: backend.token
};
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
// Create wmill.yaml with includeSettings
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes:
- "**"
includeSettings: true`, "utf-8");
// First pull to get baseline settings
const pullResult = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
expect(pullResult.code).toEqual(0);
// Read existing settings and add datatable config
const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8");
const existingSettings = parse(settingsContent) as Record<string, unknown>;
existingSettings.datatable = {
datatables: {
analytics: {
database: {
resource_path: "u/admin/analytics_db",
resource_type: "postgresql"
}
}
}
};
await writeFile(`${tempDir}/settings.yaml`, stringify(existingSettings), "utf-8");
// Push the modified settings
const pushResult = await backend.runCLICommand(['sync', 'push', '--yes'], tempDir);
expect(pushResult.code).toEqual(0);
// Verify the backend has the updated datatable config
const settingsResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/workspaces/get_settings`
);
expect(settingsResp.ok).toBe(true);
const settings = await settingsResp.json();
expect(settings.datatable).toBeDefined();
expect(settings.datatable.datatables).toBeDefined();
expect(settings.datatable.datatables.analytics).toBeDefined();
expect(settings.datatable.datatables.analytics.database.resource_path).toEqual("u/admin/analytics_db");
expect(settings.datatable.datatables.analytics.database.resource_type).toEqual("postgresql");
});
});
test.skipIf(shouldSkipOnCI())("Datatable config: empty/undefined doesn't cause errors", async () => {
await withTestBackend(async (backend, tempDir) => {
const testWorkspace = {
remote: backend.baseUrl,
workspaceId: backend.workspace,
name: "datatable_empty_test",
token: backend.token
};
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
// Ensure datatable config is empty/cleared on backend
const clearResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/workspaces/edit_datatable_config`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ settings: { datatables: {} } })
}
);
expect(clearResp.ok).toBe(true);
// Create wmill.yaml with includeSettings
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes:
- "**"
includeSettings: true`, "utf-8");
// Pull should succeed even with empty datatable config
const pullResult = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
expect(pullResult.code).toEqual(0);
// Settings.yaml should exist
const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8");
expect(settingsContent.length).toBeGreaterThan(0);
// Push should also succeed
const pushResult = await backend.runCLICommand(['sync', 'push', '--yes'], tempDir);
expect(pushResult.code).toEqual(0);
});
});
test.skipIf(shouldSkipOnCI())("Datatable config: round-trip preserves structure", async () => {
await withTestBackend(async (backend, tempDir) => {
const testWorkspace = {
remote: backend.baseUrl,
workspaceId: backend.workspace,
name: "datatable_roundtrip_test",
token: backend.token
};
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
// Set up a complex datatable config on backend with multiple datatables
const originalConfig = {
datatables: {
users: {
database: {
resource_path: "f/shared/users_db",
resource_type: "postgresql"
}
},
logs: {
database: {
resource_path: "f/shared/logs_db",
resource_type: "instance"
}
}
}
};
const configResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/workspaces/edit_datatable_config`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ settings: originalConfig })
}
);
expect(configResp.ok).toBe(true);
// Create wmill.yaml with includeSettings
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes:
- "**"
includeSettings: true`, "utf-8");
// Pull
const pullResult = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
expect(pullResult.code).toEqual(0);
// Push back without modification
const pushResult = await backend.runCLICommand(['sync', 'push', '--yes'], tempDir);
expect(pushResult.code).toEqual(0);
// Verify the config is preserved
const settingsResp = await backend.apiRequest!(
`/api/w/${backend.workspace}/workspaces/get_settings`
);
expect(settingsResp.ok).toBe(true);
const settings = await settingsResp.json();
expect(settings.datatable).toBeDefined();
expect(settings.datatable.datatables.users).toBeDefined();
expect(settings.datatable.datatables.logs).toBeDefined();
expect(settings.datatable.datatables.users.database.resource_path).toEqual("f/shared/users_db");
expect(settings.datatable.datatables.logs.database.resource_type).toEqual("instance");
});
});
@@ -0,0 +1,498 @@
/**
* Unit tests for failure_module and preprocessor_module inline script
* extraction (pull) and replacement (push).
*
* These tests verify that rawscript content in failure_module and
* preprocessor_module is correctly extracted to !inline references
* and resolved back, matching the existing behavior for regular modules.
*/
import { expect, test, describe } from "bun:test";
import { extractInlineScripts, extractCurrentMapping } from "../windmill-utils-internal/src/inline-scripts/extractor.ts";
import { replaceInlineScripts } from "../windmill-utils-internal/src/inline-scripts/replacer.ts";
import { newPathAssigner } from "../windmill-utils-internal/src/path-utils/path-assigner.ts";
import type { FlowModule } from "../windmill-utils-internal/src/gen/types.gen.ts";
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function makeRawscriptModule(
id: string,
content: string,
language: "bun" | "python3" | "deno" = "bun",
lock?: string,
): FlowModule {
return {
id,
value: {
type: "rawscript" as const,
content,
language,
lock: lock,
input_transforms: {},
},
};
}
const noopLogger = {
info: () => {},
error: () => {},
};
// ---------------------------------------------------------------------------
// extractInlineScripts — PULL direction
// ---------------------------------------------------------------------------
describe("extractInlineScripts for failure_module / preprocessor_module", () => {
test("extracts rawscript from failure_module wrapped in array", () => {
const failureModule = makeRawscriptModule(
"failure",
'export function main() { throw new Error("handler"); }',
"bun",
);
const scripts = extractInlineScripts([failureModule], {}, "/", "bun");
expect(scripts.length).toBeGreaterThanOrEqual(1);
const script = scripts.find((s) => !s.is_lock);
expect(script).toBeDefined();
expect(script!.content).toBe(
'export function main() { throw new Error("handler"); }',
);
// The module content should have been replaced with an !inline reference
expect(failureModule.value.content).toStartWith("!inline ");
});
test("extracts rawscript from preprocessor_module wrapped in array", () => {
const preprocessorModule = makeRawscriptModule(
"preprocessor",
"export function main() { return {}; }",
"python3",
);
const scripts = extractInlineScripts(
[preprocessorModule],
{},
"/",
"bun",
);
expect(scripts.length).toBeGreaterThanOrEqual(1);
const script = scripts.find((s) => !s.is_lock);
expect(script).toBeDefined();
expect(script!.content).toBe("export function main() { return {}; }");
expect(script!.language).toBe("python3");
expect(preprocessorModule.value.content).toStartWith("!inline ");
});
test("extracts lock alongside content", () => {
const mod = makeRawscriptModule(
"failure",
"console.log('hi')",
"bun",
"some-lock-content",
);
const scripts = extractInlineScripts([mod], {}, "/", "bun");
const contentScript = scripts.find((s) => !s.is_lock);
const lockScript = scripts.find((s) => s.is_lock);
expect(contentScript).toBeDefined();
expect(lockScript).toBeDefined();
expect(lockScript!.content).toBe("some-lock-content");
expect((mod.value as any).lock).toStartWith("!inline ");
});
test("shared pathAssigner prevents collisions when summaries match", () => {
// If a regular module and failure_module share the same summary,
// a shared PathAssigner deduplicates via its internal counter.
const regular = makeRawscriptModule("a", "code_a", "bun");
regular.summary = "my step";
const failure = makeRawscriptModule("failure", "code_failure", "bun");
failure.summary = "my step"; // same summary — would collide without shared assigner
const assigner = newPathAssigner("bun");
const scripts1 = extractInlineScripts([regular], {}, "/", "bun", assigner);
const scripts2 = extractInlineScripts([failure], {}, "/", "bun", assigner);
const allPaths = [...scripts1, ...scripts2]
.filter((s) => !s.is_lock)
.map((s) => s.path);
// All paths should be unique despite identical summaries
expect(allPaths.length).toBe(2);
expect(new Set(allPaths).size).toBe(2);
});
test("without shared pathAssigner, identical summaries produce duplicate paths", () => {
// Demonstrates the problem that sharing a PathAssigner solves.
const regular = makeRawscriptModule("a", "code_a", "bun");
regular.summary = "my step";
const failure = makeRawscriptModule("failure", "code_failure", "bun");
failure.summary = "my step";
// Separate assigners — each starts with a fresh counter
const scripts1 = extractInlineScripts([regular], {}, "/", "bun");
const scripts2 = extractInlineScripts([failure], {}, "/", "bun");
const allPaths = [...scripts1, ...scripts2]
.filter((s) => !s.is_lock)
.map((s) => s.path);
// Without a shared assigner, the paths collide
expect(allPaths.length).toBe(2);
expect(new Set(allPaths).size).toBe(1); // both got the same path
});
test("skips non-rawscript failure_module (identity type)", () => {
const identityModule: FlowModule = {
id: "failure",
value: { type: "identity" as any },
};
const scripts = extractInlineScripts([identityModule], {}, "/", "bun");
expect(scripts).toEqual([]);
});
});
// ---------------------------------------------------------------------------
// replaceInlineScripts — PUSH direction
// ---------------------------------------------------------------------------
describe("replaceInlineScripts for failure_module / preprocessor_module", () => {
test("resolves !inline reference back to file content", async () => {
const failureModule = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
);
const files: Record<string, string> = {
"failure.inline_script.ts": 'export function main() { return "error handled"; }',
};
await replaceInlineScripts(
[failureModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(failureModule.value.content).toBe(
'export function main() { return "error handled"; }',
);
});
test("resolves !inline reference for preprocessor_module", async () => {
const preprocessorModule = makeRawscriptModule(
"preprocessor",
"!inline preprocessor.inline_script.py",
"python3",
);
const files: Record<string, string> = {
"preprocessor.inline_script.py": "def main(): return {}",
};
await replaceInlineScripts(
[preprocessorModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(preprocessorModule.value.content).toBe("def main(): return {}");
});
test("resolves !inline lock reference", async () => {
const mod = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
"!inline failure.inline_script.lock",
);
const files: Record<string, string> = {
"failure.inline_script.ts": "code here",
"failure.inline_script.lock": "lock-data-here",
};
await replaceInlineScripts(
[mod],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(mod.value.content).toBe("code here");
expect((mod.value as any).lock).toBe("lock-data-here");
});
test("leaves non-inline content untouched", async () => {
const mod = makeRawscriptModule(
"failure",
"export function main() { return 1; }",
"bun",
);
await replaceInlineScripts(
[mod],
async () => {
throw new Error("should not be called");
},
noopLogger,
"/tmp/test/",
"/",
);
expect(mod.value.content).toBe("export function main() { return 1; }");
});
});
// ---------------------------------------------------------------------------
// Round-trip: extract then replace
// ---------------------------------------------------------------------------
describe("round-trip extract → replace for failure_module / preprocessor_module", () => {
test("failure_module content survives extract + replace", async () => {
const originalContent = 'export function main(error: any) {\n console.error(error);\n return { handled: true };\n}';
const failureModule = makeRawscriptModule(
"failure",
originalContent,
"bun",
);
// PULL: extract inline scripts (mutates module in place)
const extracted = extractInlineScripts([failureModule], {}, "/", "bun");
expect(failureModule.value.content).toStartWith("!inline ");
// Build a virtual filesystem from extracted scripts
const files: Record<string, string> = {};
for (const s of extracted) {
files[s.path] = s.content;
}
// PUSH: replace inline references back
await replaceInlineScripts(
[failureModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(failureModule.value.content).toBe(originalContent);
});
test("preprocessor_module content survives extract + replace", async () => {
const originalContent = "def main():\n return {\"preprocessed\": True}";
const preprocessorModule = makeRawscriptModule(
"preprocessor",
originalContent,
"python3",
);
const extracted = extractInlineScripts(
[preprocessorModule],
{},
"/",
"bun",
);
expect(preprocessorModule.value.content).toStartWith("!inline ");
const files: Record<string, string> = {};
for (const s of extracted) {
files[s.path] = s.content;
}
await replaceInlineScripts(
[preprocessorModule],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(preprocessorModule.value.content).toBe(originalContent);
});
test("failure_module with lock survives extract + replace", async () => {
const originalContent = "export function main() { return 42; }";
const originalLock = "package-lock-contents-here";
const mod = makeRawscriptModule(
"failure",
originalContent,
"bun",
originalLock,
);
const extracted = extractInlineScripts([mod], {}, "/", "bun");
const files: Record<string, string> = {};
for (const s of extracted) {
files[s.path] = s.content;
}
await replaceInlineScripts(
[mod],
async (path) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
},
noopLogger,
"/tmp/test/",
"/",
);
expect(mod.value.content).toBe(originalContent);
expect((mod.value as any).lock).toBe(originalLock);
});
test("full flow with modules + failure_module + preprocessor_module round-trips", async () => {
const regularContent = "export function main() { return 'step1'; }";
const failureContent = "export function main(e: any) { return e; }";
const preprocessorContent = "def main():\n pass";
const modules = [makeRawscriptModule("a", regularContent, "bun")];
const failureModule = makeRawscriptModule("failure", failureContent, "bun");
const preprocessorModule = makeRawscriptModule("preprocessor", preprocessorContent, "python3");
// Extract all (mimicking sync.ts pull logic)
const allExtracted = [
...extractInlineScripts(modules, {}, "/", "bun"),
...extractInlineScripts([failureModule], {}, "/", "bun"),
...extractInlineScripts([preprocessorModule], {}, "/", "bun"),
];
// All modules should now have !inline references
expect(modules[0].value.content).toStartWith("!inline ");
expect(failureModule.value.content).toStartWith("!inline ");
expect(preprocessorModule.value.content).toStartWith("!inline ");
// All paths should be unique
const paths = allExtracted.filter((s) => !s.is_lock).map((s) => s.path);
expect(new Set(paths).size).toBe(paths.length);
// Build filesystem
const files: Record<string, string> = {};
for (const s of allExtracted) {
files[s.path] = s.content;
}
const fileReader = async (path: string) => {
if (!(path in files)) throw new Error(`File not found: ${path}`);
return files[path];
};
// Replace all (mimicking flow.ts push logic)
await replaceInlineScripts(modules, fileReader, noopLogger, "/tmp/", "/");
await replaceInlineScripts([failureModule], fileReader, noopLogger, "/tmp/", "/");
await replaceInlineScripts([preprocessorModule], fileReader, noopLogger, "/tmp/", "/");
expect(modules[0].value.content).toBe(regularContent);
expect(failureModule.value.content).toBe(failureContent);
expect(preprocessorModule.value.content).toBe(preprocessorContent);
});
});
// ---------------------------------------------------------------------------
// extractCurrentMapping
// ---------------------------------------------------------------------------
describe("extractCurrentMapping for failure_module / preprocessor_module", () => {
test("extracts mapping from failure_module via optional param", () => {
const failureModule: FlowModule = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
);
const mapping = extractCurrentMapping(
undefined,
{},
failureModule,
undefined,
);
expect(mapping["failure"]).toBe("failure.inline_script.ts");
});
test("extracts mapping from preprocessor_module via optional param", () => {
const preprocessorModule: FlowModule = makeRawscriptModule(
"preprocessor",
"!inline preprocessor.inline_script.py",
"python3",
);
const mapping = extractCurrentMapping(
undefined,
{},
undefined,
preprocessorModule,
);
expect(mapping["preprocessor"]).toBe("preprocessor.inline_script.py");
});
test("extracts mapping from modules + failure + preprocessor combined", () => {
const modules: FlowModule[] = [
makeRawscriptModule("a", "!inline a.inline_script.ts", "bun"),
];
const failureModule = makeRawscriptModule(
"failure",
"!inline failure.inline_script.ts",
"bun",
);
const preprocessorModule = makeRawscriptModule(
"preprocessor",
"!inline preprocessor.inline_script.py",
"python3",
);
const mapping = extractCurrentMapping(
modules,
{},
failureModule,
preprocessorModule,
);
expect(mapping["a"]).toBe("a.inline_script.ts");
expect(mapping["failure"]).toBe("failure.inline_script.ts");
expect(mapping["preprocessor"]).toBe("preprocessor.inline_script.py");
});
test("ignores non-inline content in failure_module", () => {
const failureModule = makeRawscriptModule(
"failure",
"export function main() {}",
"bun",
);
const mapping = extractCurrentMapping(
undefined,
{},
failureModule,
undefined,
);
expect(mapping["failure"]).toBeUndefined();
});
});
+425
View File
@@ -1,6 +1,7 @@
import { expect, test } from "bun:test";
import { mkdir, writeFile } from "node:fs/promises";
import { withTestBackend } from "./test_backend.ts";
import { shouldSkipOnCI } from "./cargo_backend.ts";
// =============================================================================
// PREVIEW COMMAND INTEGRATION TESTS
@@ -27,6 +28,7 @@ async function createWmillConfig(
relative_path: string;
includes?: string[];
format?: "cjs" | "esm";
customBundler?: string;
assets?: Array<{ from: string; to: string }>;
}>;
}
@@ -44,6 +46,9 @@ async function createWmillConfig(
if (cb.format) {
yamlContent += ` format: ${cb.format}\n`;
}
if (cb.customBundler) {
yamlContent += ` customBundler: ${JSON.stringify(cb.customBundler)}\n`;
}
if (cb.assets && cb.assets.length > 0) {
yamlContent += " assets:\n";
for (const asset of cb.assets) {
@@ -122,6 +127,39 @@ schema:
await writeFile(`${dir}/flow.yaml`, flowYaml, "utf-8");
}
async function createPathScriptFlow(
tempDir: string,
flowPath: string,
options: {
summary: string;
scriptPath: string;
inputTransforms?: string;
}
): Promise<void> {
const dir = `${tempDir}/${flowPath}`;
await mkdir(dir, { recursive: true });
const inputTransforms = options.inputTransforms
? ` input_transforms:\n${options.inputTransforms}`
: " input_transforms: {}\n";
const flowYaml = `summary: "${options.summary}"
description: "Test flow"
value:
modules:
- id: "a"
value:
type: "script"
path: "${options.scriptPath}"
${inputTransforms}
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
`;
await writeFile(`${dir}/flow.yaml`, flowYaml, "utf-8");
}
// =============================================================================
// SCRIPT PREVIEW TESTS
// =============================================================================
@@ -359,6 +397,116 @@ schema:
});
});
// =============================================================================
// SCRIPT WITH MODULES PREVIEW TESTS
// =============================================================================
test("script preview: script with modules (taskScript pattern)", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, { defaultTs: "bun" });
// Create the main script that uses taskScript to call a module
await createScript(
tempDir,
"f/test/wac_script.ts",
`import { task, taskScript, workflow } from "windmill-client";
const helper = taskScript("./helper.ts");
const process = task(async (x: string): Promise<string> => {
return \`processed: \${x}\`;
});
export const main = workflow(async (x: string = "test") => {
const a = await process(x);
const b = await helper({ a });
return { processed: a, helper_result: b };
});`
);
// Create the module file in __mod/ folder
const modDir = `${tempDir}/f/test/wac_script__mod`;
await mkdir(modDir, { recursive: true });
await writeFile(
`${modDir}/helper.ts`,
`export function main(a: string): string {
return \`helper got: \${a}\`;
}`,
"utf-8"
);
const result = await backend.runCLICommand(
["script", "preview", "f/test/wac_script.ts"],
tempDir
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("processed: test");
expect(output).toContain("helper got:");
});
});
test("script preview: script with modules (folder layout)", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, { defaultTs: "bun" });
// Create folder layout: my_script__mod/script.ts + my_script__mod/helper.ts
const modDir = `${tempDir}/f/test/folder_wac__mod`;
await mkdir(modDir, { recursive: true });
// Entry point script
await writeFile(
`${modDir}/script.ts`,
`import { task, taskScript, workflow } from "windmill-client";
const helper = taskScript("./helper.ts");
export const main = workflow(async (name: string = "World") => {
const result = await helper({ name });
return { greeting: result };
});`,
"utf-8"
);
// Module file
await writeFile(
`${modDir}/helper.ts`,
`export function main(name: string): string {
return \`Hello from module, \${name}!\`;
}`,
"utf-8"
);
// Script metadata
await writeFile(
`${modDir}/script.yaml`,
`summary: "Folder layout WAC script"
description: "Test"
lock: ""
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties:
name:
type: string
default: "World"
required: []
`,
"utf-8"
);
const result = await backend.runCLICommand(
["script", "preview", `f/test/folder_wac__mod/script.ts`],
tempDir
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("Hello from module, World!");
});
});
// =============================================================================
// FLOW PREVIEW TESTS
// =============================================================================
@@ -381,3 +529,280 @@ test("flow preview: simple flow", async () => {
});
});
test("flow preview: uses local PathScript by default and remote PathScript with --remote", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, { defaultTs: "bun" });
await createScript(
tempDir,
"f/test/helper_script.ts",
`export function main(name: string = "World") { return \`Remote script says: \${name}!\`; }`
);
const pushResult = await backend.runCLICommand(
["script", "push", "f/test/helper_script.ts"],
tempDir
);
expect(pushResult.code).toEqual(0);
await writeFile(
`${tempDir}/f/test/helper_script.ts`,
`export function main(name: string = "World") { return \`Local script says: \${name}!\`; }`,
"utf-8"
);
await createPathScriptFlow(tempDir, "f/test/path_flow.flow", {
summary: "Flow with PathScript",
scriptPath: "f/test/helper_script",
inputTransforms: ` name:
type: "static"
value: "PathTest"
`,
});
const localResult = await backend.runCLICommand(
["flow", "preview", "f/test/path_flow.flow"],
tempDir
);
expect(localResult.code).toEqual(0);
expect(localResult.stdout + localResult.stderr).toContain(
"Local script says: PathTest!"
);
expect(localResult.stdout + localResult.stderr).toContain(
"Using local PathScript files for flow preview."
);
expect(localResult.stdout + localResult.stderr).toContain(
"These workspace scripts differ from the deployed version:\n- f/test/helper_script"
);
const remoteResult = await backend.runCLICommand(
["flow", "preview", "--remote", "f/test/path_flow.flow"],
tempDir
);
expect(remoteResult.code).toEqual(0);
expect(remoteResult.stdout + remoteResult.stderr).toContain(
"Remote script says: PathTest!"
);
expect(remoteResult.stdout + remoteResult.stderr).not.toContain(
"Using local PathScript files for flow preview."
);
});
});
test.skipIf(shouldSkipOnCI())("flow preview: respects defaultTs when resolving local PathScripts", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, { defaultTs: "deno" });
await createScript(
tempDir,
"f/test/deno_helper.ts",
`export function main() { return Deno.version.deno ? "deno-runtime" : "missing"; }`
);
await createPathScriptFlow(tempDir, "f/test/deno_path_flow.flow", {
summary: "Flow with Deno PathScript",
scriptPath: "f/test/deno_helper",
});
const result = await backend.runCLICommand(
["flow", "preview", "f/test/deno_path_flow.flow"],
tempDir
);
expect(result.code).toEqual(0);
expect(result.stdout + result.stderr).toContain("deno-runtime");
});
});
test("flow preview: bundles local PathScripts with local imports", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, {
defaultTs: "bun",
codebases: [{ relative_path: "f/flow_codebase", includes: ["**"] }],
});
await mkdir(`${tempDir}/f/flow_codebase`, { recursive: true });
await writeFile(
`${tempDir}/f/flow_codebase/helper.ts`,
`export function greet(name: string): string {
return \`Hello from local flow codebase, \${name}!\`;
}`,
"utf-8"
);
await createScript(
tempDir,
"f/flow_codebase/main_script.ts",
`import { greet } from "./helper";
export function main(name: string = "World") {
return greet(name);
}`
);
await createPathScriptFlow(tempDir, "f/test/importing_path_flow.flow", {
summary: "Flow with imported PathScript",
scriptPath: "f/flow_codebase/main_script",
inputTransforms: ` name:
type: "static"
value: "FlowTest"
`,
});
const result = await backend.runCLICommand(
["flow", "preview", "f/test/importing_path_flow.flow"],
tempDir
);
expect(result.code).toEqual(0);
expect(result.stdout + result.stderr).toContain(
"Hello from local flow codebase, FlowTest!"
);
});
});
test("flow preview: customBundler handles script paths with spaces", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, {
defaultTs: "bun",
codebases: [{
relative_path: "f/codebase custom",
includes: ["f/codebase custom/**"],
customBundler: "cat",
}],
});
await createScript(
tempDir,
"f/codebase custom/custom bundler.ts",
`export function main() {
return "Custom bundler path with spaces";
}`
);
await createPathScriptFlow(tempDir, "f/test/custom_bundler_path.flow", {
summary: "Flow with customBundler path",
scriptPath: "f/codebase custom/custom bundler",
});
const result = await backend.runCLICommand(
["flow", "preview", "f/test/custom_bundler_path.flow"],
tempDir
);
expect(result.code).toEqual(0);
expect(result.stdout + result.stderr).toContain(
"Custom bundler path with spaces"
);
});
});
test("flow preview: warns when local PathScript is not deployed remotely", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, { defaultTs: "bun" });
await createScript(
tempDir,
"f/test/undeployed_helper.ts",
`export function main() { return "Local only script"; }`
);
await createPathScriptFlow(tempDir, "f/test/undeployed_path_flow.flow", {
summary: "Flow with undeployed PathScript",
scriptPath: "f/test/undeployed_helper",
});
const result = await backend.runCLICommand(
["flow", "preview", "f/test/undeployed_path_flow.flow"],
tempDir
);
expect(result.code).toEqual(0);
expect(result.stdout + result.stderr).toContain("Local only script");
expect(result.stdout + result.stderr).toContain(
"These scripts do not exist in the workspace yet:\n- f/test/undeployed_helper"
);
});
});
test("flow preview: does not warn when local and deployed PathScripts match", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, { defaultTs: "bun" });
await createScript(
tempDir,
"f/test/matching_helper.ts",
`export function main() { return "Matching script"; }`
);
const pushResult = await backend.runCLICommand(
["script", "push", "f/test/matching_helper.ts"],
tempDir
);
expect(pushResult.code).toEqual(0);
await createPathScriptFlow(tempDir, "f/test/matching_path_flow.flow", {
summary: "Flow with matching PathScript",
scriptPath: "f/test/matching_helper",
});
const result = await backend.runCLICommand(
["flow", "preview", "f/test/matching_path_flow.flow"],
tempDir
);
expect(result.code).toEqual(0);
expect(result.stdout + result.stderr).toContain("Matching script");
expect(result.stdout + result.stderr).not.toContain(
"Using local PathScript files for flow preview."
);
});
});
test("flow preview: fails loudly for asset-backed codebase scripts", async () => {
await withTestBackend(async (backend, tempDir) => {
await createWmillConfig(tempDir, {
defaultTs: "bun",
codebases: [{
relative_path: "f/codebase_tar",
includes: ["**"],
assets: [{ from: "f/codebase_tar/data.json", to: "data.json" }],
}],
});
await mkdir(`${tempDir}/f/codebase_tar`, { recursive: true });
await writeFile(
`${tempDir}/f/codebase_tar/data.json`,
JSON.stringify({ message: "Hello from asset!" }),
"utf-8"
);
await createScript(
tempDir,
"f/codebase_tar/main_script.ts",
`import * as fs from "fs";
export function main() {
const data = JSON.parse(fs.readFileSync("data.json", "utf-8"));
return data.message;
}`
);
await createPathScriptFlow(tempDir, "f/test/assets_path_flow.flow", {
summary: "Flow with asset-backed PathScript",
scriptPath: "f/codebase_tar/main_script",
});
const localResult = await backend.runCLICommand(
["flow", "preview", "f/test/assets_path_flow.flow"],
tempDir
);
expect(localResult.code).not.toEqual(0);
expect(localResult.stdout + localResult.stderr).toContain(
"requires codebase assets"
);
});
});
+53 -2
View File
@@ -107,7 +107,7 @@ async function readFileContent(filePath: string): Promise<string> {
* Create a raw app directory structure on disk
* Uses .raw_app folder suffix with raw_app.yaml metadata
*/
async function createRawAppOnDisk(appDir: string): Promise<void> {
async function createRawAppOnDisk(appDir: string, includeBackend: boolean = false): Promise<void> {
await mkdir(appDir, { recursive: true });
await mkdir(path.join(appDir, "inline_scripts"), { recursive: true });
@@ -131,6 +131,16 @@ async function createRawAppOnDisk(appDir: string): Promise<void> {
INLINE_SCRIPT_A_LOCK,
"utf-8"
);
// Optionally create backend runnable (type: inline)
if (includeBackend) {
await mkdir(path.join(appDir, "backend"), { recursive: true });
await writeFile(path.join(appDir, "backend", "query.yaml"), "type: inline\n", "utf-8");
await writeFile(path.join(appDir, "backend", "query.ts"), `export async function main(x: number): Promise<string> {
return \`Result: \${x}\`;
}
`, "utf-8");
}
}
test("Raw App: full sync workflow - push, pull, modify, push, clear, pull", async () => {
@@ -153,7 +163,7 @@ excludes: []`, "utf-8");
// Create folder structure
const appDir = path.join(tempDir, "f", "test", "my_raw_app.raw_app");
await mkdir(path.join(tempDir, "f", "test"), { recursive: true });
await createRawAppOnDisk(appDir);
await createRawAppOnDisk(appDir, true); // Include backend for metadata test
// =========================================================================
// STEP 1: Initial push - create raw app on backend
@@ -266,6 +276,47 @@ excludes: []`, "utf-8");
const pulledInlineScript = await readFileContent(inlineScriptPath);
expect(pulledInlineScript).toContain("modified:");
// =========================================================================
// STEP 7: Test that script generate-metadata does NOT process backend runnables
// =========================================================================
// Create a standalone script (should be processed by script generate-metadata)
await writeFile(path.join(tempDir, "f", "test", "standalone.ts"), `export async function main(): Promise<string> {
return "hello";
}
`, "utf-8");
// Run script generate-metadata
const metaResult1 = await backend.runCLICommand(
['script', 'generate-metadata', '--yes'],
tempDir, "raw_app_test"
);
expect(metaResult1.code).toEqual(0);
// Run generate-metadata --skip-flows --skip-apps
const metaResult2 = await backend.runCLICommand(
['generate-metadata', '--skip-flows', '--skip-apps', '--yes'],
tempDir, "raw_app_test"
);
expect(metaResult2.code).toEqual(0);
// Backend runnables should NOT have .script.yaml files
const backendDir = path.join(appDir, "backend");
expect(await fileExists(path.join(backendDir, "query.yaml"))).toBeTruthy();
expect(await fileExists(path.join(backendDir, "query.ts"))).toBeTruthy();
expect(await fileExists(path.join(backendDir, "query.script.yaml"))).toBeFalsy();
expect(await fileExists(path.join(backendDir, "query.script.lock"))).toBeFalsy();
// Bug: raw app backend files get misprocessed and create script files at wrong location
// The path f/test/my_raw_app.raw_app/backend/query.ts gets truncated at first "."
// becoming f/test/my_raw_app.script.yaml (stripping .raw_app/backend/query.ts)
expect(await fileExists(path.join(tempDir, "f", "test", "my_raw_app.script.yaml"))).toBeFalsy();
expect(await fileExists(path.join(tempDir, "f", "test", "my_raw_app.script.lock"))).toBeFalsy();
// Standalone script SHOULD have metadata
expect(await fileExists(path.join(tempDir, "f", "test", "standalone.script.yaml"))).toBeTruthy();
expect(await fileExists(path.join(tempDir, "f", "test", "standalone.script.lock"))).toBeTruthy();
});
});
+420
View File
@@ -0,0 +1,420 @@
/**
* Relative Imports Tests
*
* E2E tests for the `generate-metadata` command with relative imports:
* - Lock files correctly include transitive dependencies
* - Staleness propagates through import chains
* - Various import patterns handled correctly
*/
import { expect, test } from "bun:test";
import { writeFile, readFile, mkdir } from "node:fs/promises";
import { withTestBackend } from "./test_backend.ts";
// TODO: re-enable Python tests on CI if python feature is included by default
const isCI = process.env["CI_MINIMAL_FEATURES"] === "true";
const defaultMetadata = `summary: "Test"
schema:
type: object
properties: {}
lock: ""
`;
// =============================================================================
// Test 1: TS basic import with npm dependency propagation
// =============================================================================
test("TS: imported script's npm dep appears in importer's lock", { timeout: 60000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
const scriptA = `import { helper } from "./script_b.ts";
export async function main() { return helper(); }
`;
const scriptB = `import _ from "lodash";
export function helper() { return _.VERSION; }
`;
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
const result = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(result.code, `generate-metadata failed:\nSTDOUT: ${result.stdout}\nSTDERR: ${result.stderr}`).toBe(0);
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
expect(lockB).toContain("lodash");
expect(lockA).toContain("lodash");
});
});
// =============================================================================
// Test 2: TS chained imports - dependency propagates through chain
// =============================================================================
test("TS: chained imports propagate npm deps through entire chain", { timeout: 60000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
const scriptA = `import { utilB } from "./script_b.ts";
export async function main() { return utilB(); }
`;
const scriptB = `import { utilC } from "./script_c.ts";
export function utilB() { return utilC() + " B"; }
`;
const scriptC = `import _ from "lodash";
export function utilC() { return _.VERSION; }
`;
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC);
await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata);
const result = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(result.code).toBe(0);
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => "");
expect(lockC).toContain("lodash");
expect(lockB).toContain("lodash");
expect(lockA).toContain("lodash");
});
});
// =============================================================================
// Test 3: TS circular imports - completes without hanging, locks generated
// =============================================================================
test("TS: circular imports handled gracefully with correct locks", { timeout: 60000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
// Circular: A imports B, B imports A, B has npm dep
const scriptA = `import { funcB } from "./script_b.ts";
export function funcA() { return "A"; }
export async function main() { return funcA() + funcB(); }
`;
const scriptB = `import { funcA } from "./script_a.ts";
import _ from "lodash";
export function funcB() { return _.VERSION + funcA(); }
`;
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
const result = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(result.code).toBe(0);
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
expect(lockB).toContain("lodash");
expect(lockA).toContain("lodash");
});
});
// =============================================================================
// Test 4: Python basic import with pip dependency propagation
// =============================================================================
test.skipIf(isCI)("Python: imported script's pip dep appears in importer's lock", { timeout: 60000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
const mainPy = `from f.test.helper import helper_func
def main():
return helper_func()
`;
const helperPy = `import requests
def helper_func():
return requests.__version__
`;
await writeFile(`${tempDir}/f/test/main.py`, mainPy);
await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/helper.py`, helperPy);
await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata);
const result = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
if (result.code !== 0) {
console.log("STDOUT:", result.stdout);
console.log("STDERR:", result.stderr);
}
expect(result.code).toBe(0);
const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => "");
const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => "");
expect(lockHelper).toContain("requests");
expect(lockMain).toContain("requests");
});
});
// =============================================================================
// Test 5: Diamond dependency - A imports B and C, both import D
// =============================================================================
test.skipIf(isCI)("Python: diamond dependency pattern propagates correctly", { timeout: 60000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
// Diamond: A -> B, A -> C, B -> D, C -> D
const scriptA = `from f.test.script_b import func_b
from f.test.script_c import func_c
def main():
return func_b() + func_c()
`;
const scriptB = `from f.test.script_d import func_d
def func_b():
return "B" + func_d()
`;
const scriptC = `from f.test.script_d import func_d
def func_c():
return "C" + func_d()
`;
const scriptD = `import requests
def func_d():
return requests.__version__
`;
await writeFile(`${tempDir}/f/test/script_a.py`, scriptA);
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_b.py`, scriptB);
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_c.py`, scriptC);
await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_d.py`, scriptD);
await writeFile(`${tempDir}/f/test/script_d.script.yaml`, defaultMetadata);
const result = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(result.code).toBe(0);
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => "");
const lockD = await readFile(`${tempDir}/f/test/script_d.script.lock`, "utf-8").catch(() => "");
expect(lockD).toContain("requests");
expect(lockB).toContain("requests");
expect(lockC).toContain("requests");
expect(lockA).toContain("requests");
});
});
// =============================================================================
// Test 6: Script isolation - unrelated script not marked stale
// =============================================================================
test("Script isolation: unrelated script not affected by changes", { timeout: 120000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
// A imports B, C is isolated
const scriptA = `import { helper } from "./script_b.ts";
export async function main() { return helper(); }
`;
const scriptB = `export function helper() { return "B"; }
`;
const scriptC = `export async function main() { return "isolated"; }
`;
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC);
await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata);
// Generate initial metadata
const initial = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(initial.code).toBe(0);
// Verify all up to date
const check1 = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"],
tempDir
);
expect(check1.stdout).toContain("All metadata up-to-date");
// Change script_b
await writeFile(`${tempDir}/f/test/script_b.ts`,
`export function helper() { return "B changed"; }
`);
// script_a and script_b should be stale, script_c should NOT be mentioned
const check2 = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"],
tempDir
);
expect(check2.code).toBe(0);
expect(check2.stdout).toContain("script_b");
expect(check2.stdout).toContain("script_a");
expect(check2.stdout).not.toMatch(/script_c/);
});
});
// =============================================================================
// Test 7: Python relative imports with dot syntax
// =============================================================================
test.skipIf(isCI)("Python: relative imports with dot syntax work correctly", { timeout: 60000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/mymodule`, { recursive: true });
// Using relative import syntax
const mainPy = `from .helper import helper_func
def main():
return helper_func()
`;
const helperPy = `import requests
def helper_func():
return requests.__version__
`;
await writeFile(`${tempDir}/f/mymodule/main.py`, mainPy);
await writeFile(`${tempDir}/f/mymodule/main.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/mymodule/helper.py`, helperPy);
await writeFile(`${tempDir}/f/mymodule/helper.script.yaml`, defaultMetadata);
const result = await backend.runCLICommand(
["generate-metadata", "-i", "f/mymodule/*", "--yes"],
tempDir
);
expect(result.code).toBe(0);
const lockMain = await readFile(`${tempDir}/f/mymodule/main.script.lock`, "utf-8").catch(() => "");
const lockHelper = await readFile(`${tempDir}/f/mymodule/helper.script.lock`, "utf-8").catch(() => "");
expect(lockHelper).toContain("requests");
expect(lockMain).toContain("requests");
});
});
// =============================================================================
// Test 8: Adding new import updates importer's lock
// =============================================================================
test.skipIf(isCI)("Python: adding new import updates importer's lock correctly", { timeout: 120000 }, async () => {
await withTestBackend(async (backend, tempDir) => {
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
excludes: []`);
await mkdir(`${tempDir}/f/test`, { recursive: true });
// Initial: main imports helper, helper has no external deps
const mainPy = `from f.test.helper import helper_func
def main():
return helper_func()
`;
const helperPyInitial = `def helper_func():
return "no deps"
`;
await writeFile(`${tempDir}/f/test/main.py`, mainPy);
await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata);
await writeFile(`${tempDir}/f/test/helper.py`, helperPyInitial);
await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata);
// Generate initial locks
const initial = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(initial.code).toBe(0);
// Add new script with pip dep
const utilsPy = `import requests
def get_version():
return requests.__version__
`;
await writeFile(`${tempDir}/f/test/utils.py`, utilsPy);
await writeFile(`${tempDir}/f/test/utils.script.yaml`, defaultMetadata);
// Modify helper to import utils
const helperPyWithImport = `from f.test.utils import get_version
def helper_func():
return get_version()
`;
await writeFile(`${tempDir}/f/test/helper.py`, helperPyWithImport);
// Regenerate - main should now have requests
const afterAdd = await backend.runCLICommand(
["generate-metadata", "-i", "f/test/*", "--yes"],
tempDir
);
expect(afterAdd.code).toBe(0);
const lockUtils = await readFile(`${tempDir}/f/test/utils.script.lock`, "utf-8").catch(() => "");
const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => "");
const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => "");
expect(lockUtils).toContain("requests");
expect(lockHelper).toContain("requests");
expect(lockMain).toContain("requests");
});
});
File diff suppressed because it is too large Load Diff
+322
View File
@@ -0,0 +1,322 @@
/**
* Unit tests for replacePathScriptsWithLocal.
*
* Tests that PathScript ("script" type) modules are correctly converted
* to RawScript ("rawscript" type) using local file content during
* flow preview / dev mode.
*/
import { expect, test, describe } from "bun:test";
import { replacePathScriptsWithLocal, type LocalScriptInfo } from "../windmill-utils-internal/src/inline-scripts/replacer.ts";
import type { FlowModule } from "../windmill-utils-internal/src/gen/types.gen.ts";
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function makePathScriptModule(
id: string,
path: string,
inputTransforms: Record<string, any> = {},
tagOverride?: string,
): FlowModule {
return {
id,
value: {
type: "script" as const,
path,
input_transforms: inputTransforms,
tag_override: tagOverride,
},
};
}
function makeRawscriptModule(
id: string,
content: string,
language: "bun" | "python3" | "deno" = "bun",
): FlowModule {
return {
id,
value: {
type: "rawscript" as const,
content,
language,
input_transforms: {},
},
};
}
const noopLogger = {
info: () => {},
error: () => {},
};
// ---------------------------------------------------------------------------
// Basic conversion tests
// ---------------------------------------------------------------------------
describe("replacePathScriptsWithLocal", () => {
test("converts PathScript to RawScript when local file exists", async () => {
const module = makePathScriptModule("a", "f/scripts/my_script", {
x: { type: "static", value: 42 },
});
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/my_script") {
return {
content: 'export function main() { return "hello"; }',
language: "bun",
lock: "some-lock",
};
}
return undefined;
};
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
expect(module.value.type).toBe("rawscript");
expect((module.value as any).content).toBe('export function main() { return "hello"; }');
expect((module.value as any).language).toBe("bun");
expect((module.value as any).lock).toBe("some-lock");
expect((module.value as any).path).toBe("f/scripts/my_script");
expect((module.value as any).input_transforms).toEqual({ x: { type: "static", value: 42 } });
});
test("preserves tag_override as tag", async () => {
const module = makePathScriptModule("a", "f/scripts/tagged", {}, "my-worker");
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/tagged") {
return { content: "code", language: "python3", tag: "script-worker" };
}
return undefined;
};
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
expect(module.value.type).toBe("rawscript");
expect((module.value as any).tag).toBe("my-worker");
});
test("uses local script tag when no tag_override is set", async () => {
const module = makePathScriptModule("a", "f/scripts/tagged");
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/tagged") {
return { content: "code", language: "python3", tag: "script-worker" };
}
return undefined;
};
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
expect(module.value.type).toBe("rawscript");
expect((module.value as any).tag).toBe("script-worker");
});
test("leaves PathScript untouched when local file not found", async () => {
const module = makePathScriptModule("a", "f/scripts/remote_only");
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => undefined;
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
expect(module.value.type).toBe("script");
expect((module.value as any).path).toBe("f/scripts/remote_only");
});
test("does not affect rawscript modules", async () => {
const module = makeRawscriptModule("a", "existing code", "bun");
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => {
throw new Error("should not be called for rawscript");
};
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
expect(module.value.type).toBe("rawscript");
expect((module.value as any).content).toBe("existing code");
});
test("handles mixed PathScript and RawScript modules", async () => {
const pathModule = makePathScriptModule("a", "f/scripts/local_script");
const rawModule = makeRawscriptModule("b", "inline code", "bun");
const remoteModule = makePathScriptModule("c", "f/scripts/remote_only");
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/local_script") {
return { content: "local code", language: "python3" };
}
return undefined;
};
await replacePathScriptsWithLocal(
[pathModule, rawModule, remoteModule],
scriptReader,
noopLogger,
);
// PathScript with local file → converted
expect(pathModule.value.type).toBe("rawscript");
expect((pathModule.value as any).content).toBe("local code");
// RawScript → untouched
expect(rawModule.value.type).toBe("rawscript");
expect((rawModule.value as any).content).toBe("inline code");
// PathScript without local file → untouched
expect(remoteModule.value.type).toBe("script");
expect((remoteModule.value as any).path).toBe("f/scripts/remote_only");
});
test("handles module without lock", async () => {
const module = makePathScriptModule("a", "f/scripts/no_lock");
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/no_lock") {
return { content: "code", language: "go" };
}
return undefined;
};
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
expect(module.value.type).toBe("rawscript");
expect((module.value as any).lock).toBeUndefined();
});
});
// ---------------------------------------------------------------------------
// Nested structure tests
// ---------------------------------------------------------------------------
describe("replacePathScriptsWithLocal nested structures", () => {
test("processes modules inside forloopflow", async () => {
const innerModule = makePathScriptModule("inner", "f/scripts/loop_script");
const loopModule: FlowModule = {
id: "loop",
value: {
type: "forloopflow" as const,
iterator: { type: "static" as const, value: "" },
modules: [innerModule],
skip_failures: false,
},
};
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/loop_script") {
return { content: "loop code", language: "bun" };
}
return undefined;
};
await replacePathScriptsWithLocal([loopModule], scriptReader, noopLogger);
expect(innerModule.value.type).toBe("rawscript");
expect((innerModule.value as any).content).toBe("loop code");
});
test("processes modules inside whileloopflow", async () => {
const innerModule = makePathScriptModule("inner", "f/scripts/while_script");
const whileModule: FlowModule = {
id: "while",
value: {
type: "whileloopflow" as const,
modules: [innerModule],
skip_failures: false,
},
};
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/while_script") {
return { content: "while code", language: "bun" };
}
return undefined;
};
await replacePathScriptsWithLocal([whileModule], scriptReader, noopLogger);
expect(innerModule.value.type).toBe("rawscript");
expect((innerModule.value as any).content).toBe("while code");
});
test("processes modules inside branchall", async () => {
const branch1Module = makePathScriptModule("b1", "f/scripts/branch1");
const branch2Module = makePathScriptModule("b2", "f/scripts/branch2");
const branchAllModule: FlowModule = {
id: "branches",
value: {
type: "branchall" as const,
branches: [
{ modules: [branch1Module], skip_failure: false },
{ modules: [branch2Module], skip_failure: false },
],
},
};
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/branch1") {
return { content: "branch1 code", language: "bun" };
}
if (scriptPath === "f/scripts/branch2") {
return { content: "branch2 code", language: "python3" };
}
return undefined;
};
await replacePathScriptsWithLocal([branchAllModule], scriptReader, noopLogger);
expect(branch1Module.value.type).toBe("rawscript");
expect((branch1Module.value as any).content).toBe("branch1 code");
expect(branch2Module.value.type).toBe("rawscript");
expect((branch2Module.value as any).content).toBe("branch2 code");
expect((branch2Module.value as any).language).toBe("python3");
});
test("processes modules inside branchone (branches + default)", async () => {
const branchModule = makePathScriptModule("b1", "f/scripts/branch");
const defaultModule = makePathScriptModule("d1", "f/scripts/default");
const branchOneModule: FlowModule = {
id: "branchone",
value: {
type: "branchone" as const,
branches: [
{ modules: [branchModule], expr: "true" },
],
default: [defaultModule],
},
};
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
if (scriptPath === "f/scripts/branch") {
return { content: "branch code", language: "bun" };
}
if (scriptPath === "f/scripts/default") {
return { content: "default code", language: "bash" };
}
return undefined;
};
await replacePathScriptsWithLocal([branchOneModule], scriptReader, noopLogger);
expect(branchModule.value.type).toBe("rawscript");
expect((branchModule.value as any).content).toBe("branch code");
expect(defaultModule.value.type).toBe("rawscript");
expect((defaultModule.value as any).content).toBe("default code");
expect((defaultModule.value as any).language).toBe("bash");
});
test("handles empty modules array", async () => {
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => undefined;
// Should not throw
await replacePathScriptsWithLocal([], scriptReader, noopLogger);
});
test("handles module with undefined value gracefully", async () => {
const module: FlowModule = { id: "x", value: undefined as any };
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => undefined;
// Should not throw
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
});
});
+100
View File
@@ -15,6 +15,7 @@ import {
isAppPath,
isRawAppPath,
isFolderResourcePath,
isFolderResourcePathAnyFormat,
detectFolderResourceType,
isRawAppBackendPath,
isAppInlineScriptPath,
@@ -32,6 +33,8 @@ import {
isRawAppFolderMetadataFile,
getDeleteSuffix,
transformJsonPathToDir,
isModuleEntryPoint,
getScriptBasePathFromModulePath,
} from "../src/utils/resource_folders.ts";
import { removeWorkerPrefix } from "../src/commands/worker-groups/worker-groups.ts";
@@ -212,6 +215,39 @@ describe("isFolderResourcePath", () => {
});
});
// This is the bug that isFolderResourcePathAnyFormat fixes:
// when nonDottedPaths is false (default), isFolderResourcePath misses non-dotted paths
// like "f/my_raw__raw_app/backend/a.ts", causing raw app backend scripts to leak
// into the standalone script list during generate-metadata.
describe("isFolderResourcePathAnyFormat", () => {
test("detects non-dotted paths even when global setting is dotted", () => {
setNonDottedPaths(false);
expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true);
expect(isFolderResourcePathAnyFormat("f/my_flow__flow/step.ts")).toBe(true);
expect(isFolderResourcePathAnyFormat("f/dashboard__app/inline.ts")).toBe(true);
});
test("detects dotted paths even when global setting is non-dotted", () => {
setNonDottedPaths(true);
expect(isFolderResourcePathAnyFormat("f/my_raw.raw_app/backend/a.ts")).toBe(true);
expect(isFolderResourcePathAnyFormat("f/my_flow.flow/step.ts")).toBe(true);
expect(isFolderResourcePathAnyFormat("f/dashboard.app/inline.ts")).toBe(true);
});
test("rejects non-folder-resource paths", () => {
expect(isFolderResourcePathAnyFormat("f/my_script.ts")).toBe(false);
expect(isFolderResourcePathAnyFormat("f/var.variable.yaml")).toBe(false);
});
test("confirms isFolderResourcePath fails for mismatched format (the bug)", () => {
setNonDottedPaths(false);
// isFolderResourcePath misses non-dotted paths when setting is dotted
expect(isFolderResourcePath("f/my_raw__raw_app/backend/a.ts")).toBe(false);
// isFolderResourcePathAnyFormat catches it
expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true);
});
});
describe("detectFolderResourceType", () => {
test("detects flow type", () => {
expect(detectFolderResourceType("f/x.flow/flow.yaml")).toBe("flow");
@@ -504,6 +540,70 @@ describe("transformJsonPathToDir", () => {
// removeWorkerPrefix (from worker-groups.ts)
// =============================================================================
// =============================================================================
// Module Path Functions
// =============================================================================
describe("isModuleEntryPoint", () => {
test("detects entry point files in __mod folders", () => {
expect(isModuleEntryPoint("f/my_script__mod/script.ts")).toBe(true);
expect(isModuleEntryPoint("u/admin/tool__mod/script.py")).toBe(true);
expect(isModuleEntryPoint("f/nested/path/script__mod/script.go")).toBe(true);
});
test("rejects non-entry-point files in __mod folders", () => {
expect(isModuleEntryPoint("f/my_script__mod/helper.ts")).toBe(false);
expect(isModuleEntryPoint("f/my_script__mod/utils/math.py")).toBe(false);
expect(isModuleEntryPoint("f/my_script__mod/helper.lock")).toBe(false);
});
test("rejects entry point files in subdirectories of __mod", () => {
expect(isModuleEntryPoint("f/my_script__mod/sub/script.ts")).toBe(false);
});
test("rejects paths without __mod", () => {
expect(isModuleEntryPoint("f/my_script.ts")).toBe(false);
expect(isModuleEntryPoint("f/script.ts")).toBe(false);
});
test("handles windows-style separators", () => {
expect(isModuleEntryPoint("f\\my_script__mod\\script.ts")).toBe(true);
expect(isModuleEntryPoint("f\\my_script__mod\\helper.ts")).toBe(false);
});
test("matches any extension for script.*", () => {
expect(isModuleEntryPoint("f/x__mod/script.yaml")).toBe(true);
expect(isModuleEntryPoint("f/x__mod/script.json")).toBe(true);
expect(isModuleEntryPoint("f/x__mod/script.lock")).toBe(true);
expect(isModuleEntryPoint("f/x__mod/script.sh")).toBe(true);
});
});
describe("getScriptBasePathFromModulePath", () => {
test("extracts base path from module entry point", () => {
expect(getScriptBasePathFromModulePath("f/my_script__mod/script.ts")).toBe("f/my_script");
expect(getScriptBasePathFromModulePath("u/admin/tool__mod/script.py")).toBe("u/admin/tool");
});
test("extracts base path from module helper files", () => {
expect(getScriptBasePathFromModulePath("f/my_script__mod/helper.ts")).toBe("f/my_script");
expect(getScriptBasePathFromModulePath("f/my_script__mod/utils/math.py")).toBe("f/my_script");
});
test("extracts base path from nested script paths", () => {
expect(getScriptBasePathFromModulePath("f/deeply/nested/script__mod/helper.ts")).toBe("f/deeply/nested/script");
});
test("returns undefined for non-module paths", () => {
expect(getScriptBasePathFromModulePath("f/my_script.ts")).toBeUndefined();
expect(getScriptBasePathFromModulePath("f/my_flow.flow/flow.yaml")).toBeUndefined();
});
test("handles windows-style separators", () => {
expect(getScriptBasePathFromModulePath("f\\my_script__mod\\helper.ts")).toBe("f/my_script");
});
});
describe("removeWorkerPrefix", () => {
test("removes worker__ prefix", () => {
expect(removeWorkerPrefix("worker__default")).toBe("default");
+323
View File
@@ -0,0 +1,323 @@
/**
* Unit tests for script module utilities.
* These tests require no backend — they test standalone logic.
*/
import { expect, test, describe, beforeEach, afterEach } from "bun:test";
import * as fs from "node:fs";
import * as path from "node:path";
import * as os from "node:os";
import {
getModuleFolderSuffix,
isScriptModulePath,
buildModuleFolderPath,
isModuleEntryPoint,
getScriptBasePathFromModulePath,
} from "../src/utils/resource_folders.ts";
import {
writeModulesToDisk,
readModulesFromDisk,
} from "../src/commands/script/script.ts";
import { getTypeStrFromPath } from "../src/types.ts";
// =============================================================================
// Module Path Utilities
// =============================================================================
describe("getModuleFolderSuffix", () => {
test("returns __mod", () => {
expect(getModuleFolderSuffix()).toBe("__mod");
});
});
describe("isScriptModulePath", () => {
test("detects module paths", () => {
expect(isScriptModulePath("f/my_script__mod/helper.ts")).toBe(true);
expect(isScriptModulePath("u/admin/script__mod/utils/math.py")).toBe(true);
});
test("rejects non-module paths", () => {
expect(isScriptModulePath("f/my_script.ts")).toBe(false);
expect(isScriptModulePath("f/my_script__mod")).toBe(false); // no trailing /
expect(isScriptModulePath("f/my_flow__flow/flow.yaml")).toBe(false);
});
test("handles windows-style separators", () => {
expect(isScriptModulePath("f\\my_script__mod\\helper.ts")).toBe(true);
});
});
describe("buildModuleFolderPath", () => {
test("appends __mod suffix", () => {
expect(buildModuleFolderPath("f/my_script")).toBe("f/my_script__mod");
expect(buildModuleFolderPath("u/admin/tool")).toBe("u/admin/tool__mod");
});
});
// =============================================================================
// writeModulesToDisk / readModulesFromDisk round-trip
// =============================================================================
describe("module read/write round-trip", () => {
let tempDir: string;
beforeEach(() => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-module-test-"));
});
afterEach(() => {
fs.rmSync(tempDir, { recursive: true, force: true });
});
test("writes and reads back a simple module", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
const modules = {
"helper.ts": {
content: 'export function greet() { return "hi"; }\n',
language: "bun" as const,
},
};
await writeModulesToDisk(moduleFolderPath, modules, "bun");
// Verify file exists on disk
expect(fs.existsSync(path.join(moduleFolderPath, "helper.ts"))).toBe(true);
// Read back
const result = await readModulesFromDisk(moduleFolderPath, "bun", false);
expect(result).toBeDefined();
expect(Object.keys(result!)).toEqual(["helper.ts"]);
expect(result!["helper.ts"].content).toBe('export function greet() { return "hi"; }\n');
expect(result!["helper.ts"].language).toBe("bun");
});
test("writes and reads back module with lock file", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
const modules = {
"helper.py": {
content: "def greet():\n return 'hi'\n",
language: "python3" as const,
lock: "requests==2.31.0\n",
},
};
await writeModulesToDisk(moduleFolderPath, modules, undefined);
// Verify lock file exists
expect(fs.existsSync(path.join(moduleFolderPath, "helper.lock"))).toBe(true);
// Read back
const result = await readModulesFromDisk(moduleFolderPath, undefined, false);
expect(result).toBeDefined();
expect(result!["helper.py"].lock).toBe("requests==2.31.0\n");
});
test("writes and reads back nested module paths", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
const modules = {
"utils/math.py": {
content: "def add(a, b):\n return a + b\n",
language: "python3" as const,
},
"utils/__init__.py": {
content: "",
language: "python3" as const,
},
};
await writeModulesToDisk(moduleFolderPath, modules, undefined);
// Verify nested structure
expect(fs.existsSync(path.join(moduleFolderPath, "utils", "math.py"))).toBe(true);
expect(fs.existsSync(path.join(moduleFolderPath, "utils", "__init__.py"))).toBe(true);
// Read back
const result = await readModulesFromDisk(moduleFolderPath, undefined, false);
expect(result).toBeDefined();
expect(Object.keys(result!).sort()).toEqual(["utils/__init__.py", "utils/math.py"]);
expect(result!["utils/math.py"].content).toBe("def add(a, b):\n return a + b\n");
});
test("returns undefined for non-existent folder", async () => {
const result = await readModulesFromDisk(path.join(tempDir, "nonexistent__mod"), undefined, false);
expect(result).toBeUndefined();
});
test("returns undefined for empty folder", async () => {
const emptyFolder = path.join(tempDir, "empty__mod");
fs.mkdirSync(emptyFolder);
const result = await readModulesFromDisk(emptyFolder, undefined, false);
expect(result).toBeUndefined();
});
test("multiple modules of different languages", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
const modules = {
"helper.ts": {
content: "export const x = 1;\n",
language: "bun" as const,
},
"other.ts": {
content: "export const y = 2;\n",
language: "bun" as const,
lock: "some-lock-content\n",
},
};
await writeModulesToDisk(moduleFolderPath, modules, "bun");
const result = await readModulesFromDisk(moduleFolderPath, "bun", false);
expect(result).toBeDefined();
expect(Object.keys(result!).sort()).toEqual(["helper.ts", "other.ts"]);
expect(result!["other.ts"].lock).toBe("some-lock-content\n");
expect(result!["helper.ts"].lock).toBeUndefined();
});
test("folder layout skips entry point files", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
fs.mkdirSync(moduleFolderPath, { recursive: true });
// Simulate folder layout: script.ts is the entry point, helper.ts is a module
fs.writeFileSync(path.join(moduleFolderPath, "script.ts"), "export function main() {}\n");
fs.writeFileSync(path.join(moduleFolderPath, "script.yaml"), "description: test\n");
fs.writeFileSync(path.join(moduleFolderPath, "script.lock"), "");
fs.writeFileSync(path.join(moduleFolderPath, "helper.ts"), "export const x = 1;\n");
// With folderLayout=true, entry point files should be skipped
const result = await readModulesFromDisk(moduleFolderPath, "bun", true);
expect(result).toBeDefined();
expect(Object.keys(result!)).toEqual(["helper.ts"]);
// With folderLayout=false, all files are included
const resultFlat = await readModulesFromDisk(moduleFolderPath, "bun", false);
expect(resultFlat).toBeDefined();
expect(Object.keys(resultFlat!).sort()).toContain("script.ts");
expect(Object.keys(resultFlat!).sort()).toContain("helper.ts");
});
});
// =============================================================================
// getTypeStrFromPath with module paths
// =============================================================================
describe("getTypeStrFromPath with module paths", () => {
test("recognizes module content files as scripts", () => {
expect(getTypeStrFromPath("f/my_script__mod/helper.ts")).toBe("script");
expect(getTypeStrFromPath("u/admin/tool__mod/utils/math.py")).toBe("script");
expect(getTypeStrFromPath("f/x__mod/helper.go")).toBe("script");
});
test("recognizes module lock files as scripts", () => {
// Lock files inside __mod/ should NOT throw — they are valid module files
expect(getTypeStrFromPath("f/my_script__mod/helper.lock")).toBe("script");
expect(getTypeStrFromPath("u/admin/tool__mod/utils/math.lock")).toBe("script");
});
test("recognizes module entry points as scripts", () => {
expect(getTypeStrFromPath("f/my_script__mod/script.ts")).toBe("script");
expect(getTypeStrFromPath("f/my_script__mod/script.py")).toBe("script");
});
test("recognizes module metadata files as scripts", () => {
// .yaml/.json files inside __mod/ should be recognized
expect(getTypeStrFromPath("f/my_script__mod/script.yaml")).toBe("script");
expect(getTypeStrFromPath("f/my_script__mod/script.json")).toBe("script");
});
});
// =============================================================================
// Module read/write with multiple lock files
// =============================================================================
describe("module lock file handling", () => {
let tempDir: string;
beforeEach(() => {
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-module-lock-test-"));
});
afterEach(() => {
fs.rmSync(tempDir, { recursive: true, force: true });
});
test("writes and reads multiple modules each with their own lock", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
const modules = {
"api_client.py": {
content: "import requests\ndef fetch(): pass\n",
language: "python3" as const,
lock: "requests==2.31.0\nurllib3==2.0.4\n",
},
"data_processor.py": {
content: "import pandas\ndef process(): pass\n",
language: "python3" as const,
lock: "pandas==2.1.0\nnumpy==1.25.0\n",
},
"utils.py": {
content: "def helper(): pass\n",
language: "python3" as const,
// No lock — no external deps
},
};
await writeModulesToDisk(moduleFolderPath, modules, undefined);
// Verify lock files exist only for modules with locks
expect(fs.existsSync(path.join(moduleFolderPath, "api_client.lock"))).toBe(true);
expect(fs.existsSync(path.join(moduleFolderPath, "data_processor.lock"))).toBe(true);
expect(fs.existsSync(path.join(moduleFolderPath, "utils.lock"))).toBe(false);
// Read back and verify
const result = await readModulesFromDisk(moduleFolderPath, undefined, false);
expect(result).toBeDefined();
expect(Object.keys(result!).sort()).toEqual(["api_client.py", "data_processor.py", "utils.py"]);
expect(result!["api_client.py"].lock).toBe("requests==2.31.0\nurllib3==2.0.4\n");
expect(result!["data_processor.py"].lock).toBe("pandas==2.1.0\nnumpy==1.25.0\n");
expect(result!["utils.py"].lock).toBeUndefined();
});
test("nested modules with lock files", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
const modules = {
"services/api.ts": {
content: "export function callApi() {}\n",
language: "bun" as const,
lock: "axios@1.5.0\n",
},
"services/db.ts": {
content: "export function query() {}\n",
language: "bun" as const,
lock: "pg@8.11.0\n",
},
};
await writeModulesToDisk(moduleFolderPath, modules, "bun");
// Verify nested lock files
expect(fs.existsSync(path.join(moduleFolderPath, "services", "api.lock"))).toBe(true);
expect(fs.existsSync(path.join(moduleFolderPath, "services", "db.lock"))).toBe(true);
const result = await readModulesFromDisk(moduleFolderPath, "bun", false);
expect(result).toBeDefined();
expect(result!["services/api.ts"].lock).toBe("axios@1.5.0\n");
expect(result!["services/db.ts"].lock).toBe("pg@8.11.0\n");
});
test("overwrites existing module folder on re-write", async () => {
const moduleFolderPath = path.join(tempDir, "my_script__mod");
// First write
await writeModulesToDisk(moduleFolderPath, {
"old.ts": { content: "old content\n", language: "bun" as const },
}, "bun");
expect(fs.existsSync(path.join(moduleFolderPath, "old.ts"))).toBe(true);
// Second write with different module
await writeModulesToDisk(moduleFolderPath, {
"new.ts": { content: "new content\n", language: "bun" as const },
}, "bun");
expect(fs.existsSync(path.join(moduleFolderPath, "new.ts"))).toBe(true);
// old.ts should still exist (writeModulesToDisk doesn't clean up)
// This is intentional — cleanup happens at the sync level
});
});
+57
View File
@@ -409,6 +409,63 @@ describe("script bootstrap command", () => {
});
});
test("accepts 'python' as alias for python3", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
await writeFile(
join(tempDir, "wmill.yaml"),
`defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`,
"utf-8"
);
await mkdir(join(tempDir, "f", "test"), { recursive: true });
const result = await backend.runCLICommand(
["script", "bootstrap", "f/test/py_alias_script", "python"],
tempDir
);
expect(result.code).toEqual(0);
const codeStat = await stat(join(tempDir, "f/test/py_alias_script.py"));
expect(codeStat.isFile()).toBe(true);
const metaStat = await stat(
join(tempDir, "f/test/py_alias_script.script.yaml")
);
expect(metaStat.isFile()).toBe(true);
});
});
test("creates parent directories automatically", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
await writeFile(
join(tempDir, "wmill.yaml"),
`defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`,
"utf-8"
);
// Do NOT pre-create f/test — bootstrap should create it
const result = await backend.runCLICommand(
["script", "bootstrap", "f/test/auto_dir_script", "bun"],
tempDir
);
expect(result.code).toEqual(0);
const codeStat = await stat(join(tempDir, "f/test/auto_dir_script.ts"));
expect(codeStat.isFile()).toBe(true);
const metaStat = await stat(
join(tempDir, "f/test/auto_dir_script.script.yaml")
);
expect(metaStat.isFile()).toBe(true);
});
});
test("creates Go script files", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspaceProfile(backend);
+392 -5
View File
@@ -3,6 +3,14 @@
*
* Tests the sync pull and push functionality with a simulated filesystem
* containing every kind of Windmill resource type.
*
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
* @see test_fixtures.ts - Shared local fixtures (prefer using this module for new tests)
* @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource,
* createAppWithInlineScript, createFlowWithInlineScript, etc.)
*
* This file contains: Local fixtures (should migrate to test_fixtures.ts) + createRemoteScript
* If you add new helpers, update cross-links in the files above.
*/
import { expect, test, describe } from "bun:test";
@@ -32,15 +40,19 @@ import {
isAppInlineScriptPath,
isFlowInlineScriptPath,
isRawAppBackendPath,
getModuleFolderSuffix,
} from "../src/utils/resource_folders.ts";
import { newPathAssigner } from "../windmill-utils-internal/src/path-utils/path-assigner.ts";
// =============================================================================
// Test Fixtures - Every Type of Windmill Resource
// See file header for cross-links to related helpers.
// Consider migrating these to test_fixtures.ts for reuse across tests.
// =============================================================================
/**
* Creates a mock script file structure
* Creates a mock script file structure.
* See file header for cross-links to related helpers.
*/
function createScriptFixture(
name: string,
@@ -89,7 +101,8 @@ kind: script
}
/**
* Creates a mock flow file structure
* Creates a mock flow file structure.
* See file header for cross-links to related helpers.
*/
function createFlowFixture(name: string): Record<string, { path: string; content: string }> {
const flowSuffix = getFolderSuffix("flow");
@@ -123,7 +136,8 @@ schema:
}
/**
* Creates a mock app file structure
* Creates a mock app file structure.
* See file header for cross-links to related helpers.
*/
function createAppFixture(name: string): Record<string, { path: string; content: string }> {
const appSuffix = getFolderSuffix("app");
@@ -151,7 +165,8 @@ policy:
}
/**
* Creates a mock raw_app file structure
* Creates a mock raw_app file structure.
* See file header for cross-links to related helpers.
*/
function createRawAppFixture(name: string): Record<string, { path: string; content: string }> {
const rawAppSuffix = getFolderSuffix("raw_app");
@@ -1755,6 +1770,66 @@ excludes: []
});
});
test("Integration: Sync pull with nonDottedPaths uses non-dotted inline script filenames", async () => {
await withTestBackend(async (backend, tempDir) => {
// Push a flow using default dotted paths
await writeFile(
`${tempDir}/wmill.yaml`,
`defaultTs: bun
includes:
- "**"
excludes: []
`,
"utf-8",
);
const uniqueId = Date.now();
const flowName = `f/test/nondot_pull_inline_${uniqueId}`;
const flowFixture = createFlowFixture(flowName);
await mkdir(`${tempDir}/f/test/nondot_pull_inline_${uniqueId}${getFolderSuffix("flow")}`, { recursive: true });
for (const file of Object.values(flowFixture)) {
await writeFile(`${tempDir}/${file.path}`, file.content, "utf-8");
}
const pushResult = await backend.runCLICommand(
["sync", "push", "--yes", "--includes", `f/test/nondot_pull_inline_${uniqueId}*/**`],
tempDir,
);
expect(pushResult.code).toEqual(0);
// Pull into a fresh directory with nonDottedPaths enabled
const tempDir2 = await mkdtemp(join(tmpdir(), "wmill_nondot_inline_"));
try {
await writeFile(
`${tempDir2}/wmill.yaml`,
`defaultTs: bun
nonDottedPaths: true
includes:
- "**"
excludes: []
`,
"utf-8",
);
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir2);
expect(pullResult.code).toEqual(0);
// Verify pulled files use non-dotted inline script naming
const files = await listFilesRecursive(tempDir2);
const flowFiles = files.filter((f) => f.includes(`nondot_pull_inline_${uniqueId}`));
expect(flowFiles.length > 0).toBeTruthy();
// Should use __flow folder, not .flow
expect(flowFiles.some((f) => f.includes("__flow/"))).toBeTruthy();
// No files should have .inline_script. in their name
const dottedInlineFiles = flowFiles.filter((f) => f.includes(".inline_script."));
expect(dottedInlineFiles.length).toEqual(0);
} finally {
await cleanupTempDir(tempDir2);
}
});
});
// =============================================================================
// ws_error_handler_muted Persistence Tests
// =============================================================================
@@ -1920,7 +1995,7 @@ excludes: []
import type { TestBackend } from "./test_backend.ts";
/** Create a script on the remote via API */
/** Create a script on the remote via API. See file header for cross-links. */
async function createRemoteScript(
backend: TestBackend,
scriptPath: string,
@@ -2328,3 +2403,315 @@ describe("http trigger sync", () => {
});
});
});
// =============================================================================
// Script Module Sync Tests
// =============================================================================
describe("script module sync", () => {
test.skipIf(process.platform === "win32")("push script with modules via API and pull back", async () => {
await withTestBackend(async (backend, tempDir) => {
const uniqueId = Date.now();
const scriptPath = `f/test/module_script_${uniqueId}`;
const modSuffix = getModuleFolderSuffix();
// Create a script with modules via API
const resp = await backend.apiRequest!(
`/api/w/${backend.workspace}/scripts/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: scriptPath,
content: 'import { greet } from "./helper";\nexport async function main() { return greet(); }',
language: "bun",
summary: "Script with modules",
description: "Test script with module files",
schema: {
$schema: "https://json-schema.org/draft/2020-12/schema",
type: "object",
properties: {},
required: [],
},
modules: {
"helper.ts": {
content: 'export function greet() { return "hello from module"; }\n',
language: "bun",
},
},
}),
}
);
expect(resp.status).toBeLessThan(300);
await resp.text();
await writeWmillYaml(tempDir);
// Pull the script
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
expect(pullResult.code).toEqual(0);
// Verify module files exist on disk
const files = await listFilesRecursive(tempDir);
const moduleFile = files.find(
(f) => f.includes(`module_script_${uniqueId}${modSuffix}`) && f.endsWith("helper.ts")
);
expect(moduleFile).toBeDefined();
// Scripts with modules use folder layout: main content is at __mod/script.ts
const mainFile = files.find(
(f) => f.includes(`module_script_${uniqueId}${modSuffix}`) && f.endsWith("script.ts")
);
expect(mainFile).toBeDefined();
// Verify module content
if (moduleFile) {
const content = await readFile(`${tempDir}/${moduleFile}`, "utf-8");
expect(content).toContain("greet");
expect(content).toContain("hello from module");
}
});
});
test("push script with module lock files and pull back", async () => {
await withTestBackend(async (backend, tempDir) => {
const uniqueId = Date.now();
const scriptPath = `f/test/locked_module_${uniqueId}`;
const modSuffix = getModuleFolderSuffix();
// Create a script with a module that has a lock
const resp = await backend.apiRequest!(
`/api/w/${backend.workspace}/scripts/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: scriptPath,
content: 'export async function main() { return "main"; }',
language: "bun",
summary: "Script with locked module",
schema: {
$schema: "https://json-schema.org/draft/2020-12/schema",
type: "object",
properties: {},
required: [],
},
modules: {
"dep.ts": {
content: 'import lodash from "lodash";\nexport const x = lodash.identity(1);\n',
language: "bun",
lock: "lodash@4.17.21\n",
},
},
}),
}
);
expect(resp.status).toBeLessThan(300);
await resp.text();
await writeWmillYaml(tempDir);
// Pull
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
expect(pullResult.code).toEqual(0);
// Verify lock file exists on disk
const files = await listFilesRecursive(tempDir);
const lockFile = files.find(
(f) => f.includes(`locked_module_${uniqueId}${modSuffix}`) && f.endsWith("dep.lock")
);
expect(lockFile).toBeDefined();
// Verify lock content
if (lockFile) {
const lockContent = await readFile(`${tempDir}/${lockFile}`, "utf-8");
expect(lockContent).toContain("lodash");
}
});
});
test("local script with modules pushes and round-trips", async () => {
await withTestBackend(async (backend, tempDir) => {
const uniqueId = Date.now();
const modSuffix = getModuleFolderSuffix();
await writeWmillYaml(tempDir);
// Create local script with modules using folder layout:
// f/test/modular_<id>__mod/script.ts (entry point)
// f/test/modular_<id>__mod/script.yaml (metadata)
// f/test/modular_<id>__mod/helper.ts (module file)
const scriptDir = `${tempDir}/f/test`;
const scriptName = `modular_${uniqueId}`;
const modDir = `${scriptDir}/${scriptName}${modSuffix}`;
await mkdir(modDir, { recursive: true });
// Entry point (main script content)
await writeFile(
`${modDir}/script.ts`,
'import { helper } from "./helper";\nexport async function main() { return helper(); }',
"utf-8"
);
// Script metadata
await writeFile(
`${modDir}/script.yaml`,
`summary: "Script with modules"
description: "Test"
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
is_template: false
lock: ""
kind: script
`,
"utf-8"
);
// Module file
await writeFile(
`${modDir}/helper.ts`,
'export function helper() { return "from module"; }\n',
"utf-8"
);
// Push
const pushResult = await backend.runCLICommand(["sync", "push", "--yes"], tempDir);
expect(pushResult.code).toEqual(0);
// Pull into a fresh directory to verify round-trip
const pullDir = await mkdtemp(join(tmpdir(), "windmill_module_pull_"));
try {
await writeWmillYaml(pullDir);
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], pullDir);
expect(pullResult.code).toEqual(0);
// Verify module file came back
const files = await listFilesRecursive(pullDir);
const pulledModule = files.find(
(f) => f.includes(`${scriptName}${modSuffix}`) && f.endsWith("helper.ts")
);
expect(pulledModule).toBeDefined();
if (pulledModule) {
const content = await readFile(`${pullDir}/${pulledModule}`, "utf-8");
expect(content).toContain("from module");
}
} finally {
await rm(pullDir, { recursive: true });
}
});
});
test("script with nested module paths pushes and pulls correctly", async () => {
await withTestBackend(async (backend, tempDir) => {
const uniqueId = Date.now();
const scriptPath = `f/test/nested_mod_${uniqueId}`;
const modSuffix = getModuleFolderSuffix();
// Create script with nested modules via API
const resp = await backend.apiRequest!(
`/api/w/${backend.workspace}/scripts/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: scriptPath,
content: 'export async function main() { return "main"; }',
language: "bun",
summary: "Script with nested modules",
schema: {
$schema: "https://json-schema.org/draft/2020-12/schema",
type: "object",
properties: {},
required: [],
},
modules: {
"utils/format.ts": {
content: 'export function format(s: string) { return s.trim(); }\n',
language: "bun",
},
"utils/validate.ts": {
content: 'export function validate(s: string) { return s.length > 0; }\n',
language: "bun",
},
},
}),
}
);
expect(resp.status).toBeLessThan(300);
await resp.text();
await writeWmillYaml(tempDir);
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
expect(pullResult.code).toEqual(0);
// Verify nested module files exist
const files = await listFilesRecursive(tempDir);
const formatFile = files.find(
(f) => f.includes(`nested_mod_${uniqueId}${modSuffix}`) && f.includes("utils/format.ts")
);
const validateFile = files.find(
(f) => f.includes(`nested_mod_${uniqueId}${modSuffix}`) && f.includes("utils/validate.ts")
);
expect(formatFile).toBeDefined();
expect(validateFile).toBeDefined();
});
});
test.skipIf(process.platform === "win32")("pull script with modules does not create stale metadata", async () => {
await withTestBackend(async (backend, tempDir) => {
const uniqueId = Date.now();
const scriptPath = `f/test/fresh_mod_${uniqueId}`;
// Create a script with modules on remote
const resp = await backend.apiRequest!(
`/api/w/${backend.workspace}/scripts/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path: scriptPath,
content: 'export async function main() { return "main"; }',
language: "bun",
summary: "Script with modules for freshness test",
schema: {
$schema: "https://json-schema.org/draft/2020-12/schema",
type: "object",
properties: {},
required: [],
},
modules: {
"helper.ts": {
content: 'export function help() { return true; }\n',
language: "bun",
},
},
}),
}
);
expect(resp.status).toBeLessThan(300);
await resp.text();
await writeWmillYaml(tempDir);
// Pull
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
expect(pullResult.code).toEqual(0);
// Run generate-metadata — should show up-to-date for pulled scripts
const metaResult = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir
);
expect(metaResult.code).toEqual(0);
// The script we just pulled should NOT be stale
// (it may still show other scripts as stale from seedTestData)
const output = metaResult.stdout + metaResult.stderr;
expect(output).not.toContain(`fresh_mod_${uniqueId}`);
});
});
});
+132
View File
@@ -19,6 +19,14 @@
* // ...
* });
* });
*
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
* @see test_fixtures.ts - Local file fixtures (createLocalScript, createLocalFlow, etc.)
* @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based)
*
* This file contains: API-based creation helpers (createTestApp, createTestResource,
* createAppWithInlineScript, createFlowWithInlineScript, etc.)
* If you add new helpers, update cross-links in the files above.
*/
import { CargoBackend, CargoBackendConfig } from "./cargo_backend.ts";
@@ -57,6 +65,10 @@ export interface TestBackend {
listAllApps?(): Promise<any[]>;
listAllResources?(): Promise<any[]>;
listAllVariables?(): Promise<any[]>;
// Methods for creating apps and flows with custom inline scripts
createAppWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise<void>;
createFlowWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise<void>;
}
/**
@@ -109,6 +121,7 @@ class CargoBackendAdapter implements TestBackend {
return this.backend.apiRequest(path, options);
}
/** Seeds test data via API calls. See file header for cross-links to related helpers. */
async seedTestData(): Promise<void> {
// Create test folder first
await this.createTestFolder("test");
@@ -124,6 +137,7 @@ class CargoBackendAdapter implements TestBackend {
await this.createTestApp("f/test/test_dashboard");
}
/** See file header for cross-links to related helpers. */
private async createTestApp(path: string): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, {
method: "POST",
@@ -156,6 +170,7 @@ class CargoBackendAdapter implements TestBackend {
}
}
/** See file header for cross-links to related helpers. */
private async createTestFolder(name: string): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/folders/create`, {
method: "POST",
@@ -172,6 +187,7 @@ class CargoBackendAdapter implements TestBackend {
}
}
/** See file header for cross-links to related helpers. */
private async createTestGroup(name: string): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/groups/create`, {
method: "POST",
@@ -189,6 +205,7 @@ class CargoBackendAdapter implements TestBackend {
}
/** See file header for cross-links to related helpers. */
private async createTestResource(path: string, description: string): Promise<void> {
// First ensure the folder exists
const folderPath = path.split("/").slice(0, 2).join("/"); // e.g., "f/test"
@@ -228,6 +245,7 @@ class CargoBackendAdapter implements TestBackend {
}
}
/** See file header for cross-links to related helpers. */
private async createTestVariable(path: string, value: string): Promise<void> {
const response = await this.backend.apiRequest(
`/api/w/${this.workspace}/variables/create`,
@@ -329,6 +347,88 @@ class CargoBackendAdapter implements TestBackend {
if (!response.ok) return [];
return response.json();
}
async createAppWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path,
value: {
type: "app",
grid: [
{
id: "button1",
data: {
type: "buttoncomponent",
componentInput: {
type: "runnable",
runnable: {
type: "runnableByName",
inlineScript: {
content: inlineScriptContent,
language,
},
},
},
},
},
],
hiddenInlineScripts: [],
css: {},
norefreshbar: false,
},
summary: "Test app with inline script",
policy: {
on_behalf_of: null,
on_behalf_of_email: null,
triggerables: {},
execution_mode: "viewer",
},
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Failed to create app ${path}: ${error}`);
}
await response.text();
}
async createFlowWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise<void> {
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/flows/create`, {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
path,
summary: "Test flow with inline script",
description: `Flow at ${path}`,
value: {
modules: [
{
id: "a",
value: {
type: "rawscript",
content: inlineScriptContent,
language,
input_transforms: {},
},
},
],
},
schema: {
$schema: "https://json-schema.org/draft/2020-12/schema",
type: "object",
properties: {},
required: [],
},
}),
});
if (!response.ok) {
const error = await response.text();
throw new Error(`Failed to create flow ${path}: ${error}`);
}
await response.text();
}
}
/**
@@ -567,6 +667,38 @@ export async function createNonAdminUser(
return await loginResp.text();
}
/**
* Create workspace dependencies via the API (e.g. a shared package.json for bun scripts).
*/
export async function createRemoteWorkspaceDeps(
backend: TestBackend,
language: string,
content: string,
name?: string,
): Promise<void> {
if (!backend.apiRequest) {
throw new Error("Backend does not support apiRequest");
}
const resp = await backend.apiRequest(
`/api/w/${backend.workspace}/workspace_dependencies/create`,
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
workspace_id: backend.workspace,
language,
content,
...(name ? { name } : {}),
}),
}
);
if (!resp.ok) {
throw new Error(`Failed to create workspace deps (${resp.status}): ${await resp.text()}`);
}
await resp.text();
}
// Re-export for convenience
export type { CargoBackendConfig } from "./cargo_backend.ts";
export type { ContainerConfig } from "./containerized_backend.ts";
+632
View File
@@ -0,0 +1,632 @@
/**
* Test Fixtures
*
* Shared helpers for creating test data (scripts, flows, apps, raw apps) in tests.
*
* Two types of helpers:
* - Fixture functions: Return data structures with paths and contents (no disk I/O)
* - Local creation functions: Create fixtures AND write them to disk
*
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
* @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource,
* createAppWithInlineScript, createFlowWithInlineScript, etc.)
* @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based)
*
* This file contains: Shared local fixtures (createLocalScript, createLocalFlow, etc.)
* If you add new helpers, update cross-links in the files above.
*
* @example
* // Using fixtures (data only)
* const fixture = createScriptFixture("my_script", "bun");
*
* // Using local creation (writes to disk)
* await createLocalScript(tempDir, "f/test", "my_script", "bun");
*
* @keywords createLocal, local script, local flow, local app, raw app, fixture, test data
*/
import { writeFile, mkdir } from "node:fs/promises";
import {
getFolderSuffix,
getMetadataFileName,
getModuleFolderSuffix,
getNonDottedPaths,
} from "../src/utils/resource_folders.ts";
// =============================================================================
// Fixture Types
// =============================================================================
export interface FileFixture {
path: string;
content: string;
}
export interface ScriptFixture {
contentFile: FileFixture;
metadataFile: FileFixture;
}
export interface FlowFixture {
metadata: FileFixture;
inlineScript?: FileFixture;
inlineLock?: FileFixture;
}
export interface AppFixture {
metadata: FileFixture;
}
export interface RawAppFixture {
metadata: FileFixture;
indexHtml: FileFixture;
indexJs: FileFixture;
[key: string]: FileFixture;
}
// =============================================================================
// Script Fixtures
// =============================================================================
/**
* Creates a script fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* Use this when you need fine-grained control over the script structure.
* For simple cases, use {@link createLocalScript} instead.
*
* @param name - Script name (without extension)
* @param language - Script language
* @param content - Optional custom script content
* @returns Script fixture with content and metadata files
*
* @example
* const fixture = createScriptFixture("my_script", "bun");
* const fixture = createScriptFixture("custom", "python3", "def main(): return 42");
*
* @keywords script fixture, create script, local script
*/
export function createScriptFixture(
name: string,
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
content?: string
): ScriptFixture {
const extensions: Record<string, string> = {
python3: ".py",
deno: ".ts",
bun: ".ts",
bash: ".sh",
go: ".go",
postgresql: ".sql",
};
const ext = extensions[language];
const defaultContent: Record<string, string> = {
python3: `def main():\n return "Hello from ${name}"`,
deno: `export async function main() {\n return "Hello from ${name}";\n}`,
bun: `export async function main() {\n return "Hello from ${name}";\n}`,
bash: `#!/bin/bash\necho "Hello from ${name}"`,
go: `package inner\n\nfunc main() string {\n return "Hello from ${name}"\n}`,
postgresql: `-- ${name}\nSELECT 'Hello from ${name}';`,
};
return {
contentFile: {
path: `${name}${ext}`,
content: content ?? defaultContent[language],
},
metadataFile: {
path: `${name}.script.yaml`,
content: `summary: "${name} script"
description: "A ${language} script for testing"
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
is_template: false
lock: ""
kind: script
`,
},
};
}
// =============================================================================
// Flow Fixtures
// =============================================================================
/**
* Creates a flow fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* TODO: Add optional params: language, summary, description
*
* Use this when you need fine-grained control over the flow structure.
* For simple cases, use {@link createLocalFlow} instead.
*
* @param name - Flow name
* @param inlineScriptContent - Optional custom inline script content
* @returns Flow fixture with metadata and inline script
*
* @example
* const fixture = createFlowFixture("my_flow");
*
* @keywords flow fixture, create flow, local flow
*/
export function createFlowFixture(
name: string,
inlineScriptContent?: string,
language: "bun" | "python3" = "bun",
lockContent?: string
): FlowFixture {
const flowSuffix = getFolderSuffix("flow");
const metadataFile = getMetadataFileName("flow", "yaml");
const defaultContent = language === "python3"
? `def main():\n return "Hello from flow ${name}"`
: `export async function main() {\n return "Hello from flow ${name}";\n}`;
const scriptContent = inlineScriptContent ?? defaultContent;
const langMap: Record<string, string> = { bun: "bun", python3: "python3" };
const extMap: Record<string, string> = { bun: "ts", python3: "py" };
const ext = extMap[language];
// With dotted paths (.flow), inline scripts use .inline_script suffix (a.inline_script.ts)
// With non-dotted paths (__flow), they don't (a.ts)
const inlineSuffix = getNonDottedPaths() ? "" : ".inline_script";
const scriptFile = `a${inlineSuffix}.${ext}`;
const lockFile = `a${inlineSuffix}.lock`;
const lockLine = lockContent !== undefined
? `\n lock: "!inline ${lockFile}"`
: "";
const result: FlowFixture = {
metadata: {
path: `${name}${flowSuffix}/${metadataFile}`,
content: `summary: "${name} flow"
description: "A flow for testing"
value:
modules:
- id: a
value:
type: rawscript
content: "!inline ${scriptFile}"${lockLine}
language: ${langMap[language]}
input_transforms: {}
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
`,
},
inlineScript: {
path: `${name}${flowSuffix}/${scriptFile}`,
content: scriptContent,
},
};
if (lockContent !== undefined) {
result.inlineLock = {
path: `${name}${flowSuffix}/${lockFile}`,
content: lockContent,
};
}
return result;
}
// =============================================================================
// App Fixtures
// =============================================================================
/**
* Creates an app fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* TODO: Add optional params: inlineScriptContent, summary, grid
*
* Use this when you need fine-grained control over the app structure.
* For simple cases, use {@link createLocalApp} instead.
*
* @param name - App name
* @returns App fixture with metadata
*
* @example
* const fixture = createAppFixture("my_app");
*
* @keywords app fixture, create app, local app
*/
export function createAppFixture(name: string, inlineScriptContent?: string): AppFixture {
const appSuffix = getFolderSuffix("app");
const metadataFile = getMetadataFileName("app", "yaml");
const scriptContent = inlineScriptContent ??
`export async function main() {\n return "hello from app";\n}`;
const indented = scriptContent.split("\n").join("\n ");
return {
metadata: {
path: `${name}${appSuffix}/${metadataFile}`,
content: `summary: "${name} app"
value:
type: app
grid:
- id: button1
data:
type: buttoncomponent
componentInput:
type: runnable
runnable:
type: runnableByName
inlineScript:
content: |
${indented}
language: bun
hiddenInlineScripts: []
css: {}
norefreshbar: false
policy:
on_behalf_of: null
on_behalf_of_email: null
triggerables: {}
execution_mode: viewer
`,
},
};
}
// =============================================================================
// Raw App Fixtures
// =============================================================================
/**
* Creates a raw app fixture (data structure, no disk I/O).
* See file header for cross-links to related helpers.
*
* TODO: Add optional params: inlineScriptContent, htmlContent, jsContent
*
* Raw apps are React/frontend apps with separate inline scripts.
* Use this when you need fine-grained control over the raw app structure.
* For simple cases, use {@link createLocalRawApp} instead.
*
* @param name - Raw app name
* @returns Raw app fixture with metadata and frontend files
*
* @example
* const fixture = createRawAppFixture("my_raw_app");
*
* @keywords raw app fixture, create raw app, local raw app, react app
*/
export function createRawAppFixture(name: string, inlineScriptContent?: string): RawAppFixture {
const rawAppSuffix = getFolderSuffix("raw_app");
const metadataFile = getMetadataFileName("raw_app", "yaml");
const scriptContent = inlineScriptContent ??
`export async function main(x: string) {\n return x\n}`;
return {
metadata: {
path: `${name}${rawAppSuffix}/${metadataFile}`,
content: `summary: "${name} raw app"
policy:
execution_mode: publisher
triggerables: {}
`,
},
indexHtml: {
path: `${name}${rawAppSuffix}/index.html`,
content: `<!DOCTYPE html>
<html>
<head><title>${name}</title></head>
<body><div id="root"></div></body>
</html>`,
},
indexJs: {
path: `${name}${rawAppSuffix}/index.tsx`,
content: `import React from 'react'
import { createRoot } from 'react-dom/client'
const App = () => <div><h1>${name}</h1></div>
const root = createRoot(document.getElementById('root')!)
root.render(<App/>)
`,
},
packageJson: {
path: `${name}${rawAppSuffix}/package.json`,
content: `{
"dependencies": {
"react": "19.0.0",
"react-dom": "19.0.0"
}
}`,
},
inlineScript: {
path: `${name}${rawAppSuffix}/backend/a.ts`,
content: scriptContent + "\n",
},
inlineScriptMeta: {
path: `${name}${rawAppSuffix}/backend/a.yaml`,
content: `type: inline\n`,
},
inlineScriptLock: {
path: `${name}${rawAppSuffix}/backend/a.lock`,
content: ``,
},
};
}
// =============================================================================
// Local Creation Functions (Fixture + Write to Disk)
// =============================================================================
/**
* Creates a script on the local filesystem.
* See file header for cross-links to related helpers.
*
* This is a convenience function that creates a script fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - Script name (without extension)
* @param language - Script language (default: "bun")
* @param content - Optional custom script content
*
* @example
* await createLocalScript(tempDir, "f/test", "my_script");
* await createLocalScript(tempDir, "f/test", "custom", "python3", "def main(): return 42");
*
* @keywords create local script, local script, write script, script on disk
*/
export async function createLocalScript(
tempDir: string,
path: string,
name: string,
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
content?: string
): Promise<void> {
const fixture = createScriptFixture(name, language, content);
await mkdir(`${tempDir}/${path}`, { recursive: true });
await writeFile(
`${tempDir}/${path}/${fixture.contentFile.path}`,
fixture.contentFile.content,
"utf-8"
);
await writeFile(
`${tempDir}/${path}/${fixture.metadataFile.path}`,
fixture.metadataFile.content,
"utf-8"
);
}
/**
* Creates a flow on the local filesystem.
* See file header for cross-links to related helpers.
*
* This is a convenience function that creates a flow fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - Flow name
* @param inlineScriptContent - Optional custom inline script content
*
* @example
* await createLocalFlow(tempDir, "f/test", "my_flow");
*
* @keywords create local flow, local flow, write flow, flow on disk
*/
export async function createLocalFlow(
tempDir: string,
path: string,
name: string,
inlineScriptContent?: string,
language: "bun" | "python3" = "bun",
lockContent?: string
): Promise<void> {
const fixture = createFlowFixture(name, inlineScriptContent, language, lockContent);
const flowDir = `${tempDir}/${path}/${name}${getFolderSuffix("flow")}`;
await mkdir(flowDir, { recursive: true });
for (const file of Object.values(fixture)) {
if (!file) continue;
const fullPath = `${tempDir}/${path}/${file.path}`;
await writeFile(fullPath, file.content, "utf-8");
}
}
/**
* Creates an app on the local filesystem.
* See file header for cross-links to related helpers.
*
* This is a convenience function that creates an app fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - App name
*
* @example
* await createLocalApp(tempDir, "f/test", "my_app");
*
* @keywords create local app, local app, write app, app on disk
*/
export async function createLocalApp(
tempDir: string,
path: string,
name: string,
inlineScriptContent?: string
): Promise<void> {
const fixture = createAppFixture(name, inlineScriptContent);
const appDir = `${tempDir}/${path}/${name}${getFolderSuffix("app")}`;
await mkdir(appDir, { recursive: true });
for (const file of Object.values(fixture)) {
if (!file) continue;
const fullPath = `${tempDir}/${path}/${file.path}`;
await writeFile(fullPath, file.content, "utf-8");
}
}
/**
* Creates a raw app on the local filesystem.
* See file header for cross-links to related helpers.
*
* Raw apps are React/frontend apps with separate inline scripts.
* This is a convenience function that creates a raw app fixture and writes it to disk.
*
* @param tempDir - Base directory for the test workspace
* @param path - Relative path within the workspace (e.g., "f/test")
* @param name - Raw app name
*
* @example
* await createLocalRawApp(tempDir, "f/test", "my_raw_app");
*
* @keywords create local raw app, local raw app, write raw app, raw app on disk, react app
*/
export async function createLocalRawApp(
tempDir: string,
path: string,
name: string,
inlineScriptContent?: string
): Promise<void> {
const fixture = createRawAppFixture(name, inlineScriptContent);
const rawAppSuffix = getFolderSuffix("raw_app");
const appDir = `${tempDir}/${path}/${name}${rawAppSuffix}`;
await mkdir(`${appDir}/backend`, { recursive: true });
for (const file of Object.values(fixture)) {
const fullPath = `${tempDir}/${path}/${file.path}`;
const dir = fullPath.substring(0, fullPath.lastIndexOf("/"));
await mkdir(dir, { recursive: true });
await writeFile(fullPath, file.content, "utf-8");
}
}
// =============================================================================
// Script with Modules Fixtures
// =============================================================================
export interface ModuleFile {
path: string;
content: string;
lock?: string;
}
/**
* Creates a script with module files on the local filesystem.
*
* Creates the main script, its metadata, and module files in a __mod/ folder.
* Optionally includes lock files for modules.
*
* @param tempDir - Base directory for the test workspace
* @param dir - Relative path within the workspace (e.g., "f/test")
* @param name - Script name (without extension)
* @param language - Script language (default: "bun")
* @param modules - Module files to create
*
* @example
* await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
* { path: "helper.ts", content: "export const x = 1;" },
* { path: "utils/math.ts", content: "export function add(a, b) { return a + b; }", lock: "lodash@4.0.0\n" },
* ]);
*/
export async function createLocalScriptWithModules(
tempDir: string,
dir: string,
name: string,
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
modules: ModuleFile[]
): Promise<void> {
// Create the main script
await createLocalScript(tempDir, dir, name, language);
// Create module files in __mod/ folder
const modSuffix = getModuleFolderSuffix();
const modDir = `${tempDir}/${dir}/${name}${modSuffix}`;
for (const mod of modules) {
const fullPath = `${modDir}/${mod.path}`;
const parentDir = fullPath.substring(0, fullPath.lastIndexOf("/"));
await mkdir(parentDir, { recursive: true });
await writeFile(fullPath, mod.content, "utf-8");
// Write lock file if provided
if (mod.lock) {
const baseName = mod.path.substring(0, mod.path.indexOf("."));
const lockPath = `${modDir}/${baseName}.lock`;
const lockDir = lockPath.substring(0, lockPath.lastIndexOf("/"));
await mkdir(lockDir, { recursive: true });
await writeFile(lockPath, mod.lock, "utf-8");
}
}
}
// =============================================================================
// Resource Fixtures (Variables, Resources, Schedules, etc.)
// =============================================================================
/**
* Creates a resource fixture.
*
* @keywords resource fixture, create resource
*/
export function createResourceFixture(
name: string,
resourceType: string,
value: Record<string, unknown>
): FileFixture {
return {
path: `${name}.resource.yaml`,
content: `resource_type: "${resourceType}"
value:
${Object.entries(value)
.map(([k, v]) => ` ${k}: ${JSON.stringify(v)}`)
.join("\n")}
`,
};
}
/**
* Creates a variable fixture.
*
* @keywords variable fixture, create variable
*/
export function createVariableFixture(
name: string,
value: string,
isSecret: boolean = false
): FileFixture {
return {
path: `${name}.variable.yaml`,
content: `value: "${value}"
is_secret: ${isSecret}
description: "Variable ${name} for testing"
`,
};
}
/**
* Creates a schedule fixture.
*
* @keywords schedule fixture, create schedule
*/
export function createScheduleFixture(
name: string,
scriptPath: string,
schedule: string = "0 * * * *"
): FileFixture {
return {
path: `${name}.schedule.yaml`,
content: `path: "${name}"
schedule: "${schedule}"
script_path: "${scriptPath}"
is_flow: false
args: {}
enabled: true
timezone: "UTC"
`,
};
}
+776
View File
@@ -0,0 +1,776 @@
/**
* Unified generate-metadata Command Tests
*
* Tests the new unified `generate-metadata` command that processes
* scripts, flows, and apps together.
*/
import { expect, test, describe } from "bun:test";
import { withTestBackend } from "./test_backend.ts";
import { addWorkspace } from "../workspace.ts";
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
import {
createLocalScript,
createLocalFlow,
createLocalApp,
createLocalRawApp,
createLocalScriptWithModules,
} from "./test_fixtures.ts";
/**
* Helper to set up a workspace with wmill.yaml
*/
async function setupWorkspace(
backend: any,
tempDir: string,
workspaceName: string,
nonDottedPaths = false
) {
const testWorkspace = {
remote: backend.baseUrl,
workspaceId: backend.workspace,
name: workspaceName,
token: backend.token
};
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
${nonDottedPaths ? "nonDottedPaths: true\n" : ""}includes:
- "**"
excludes: []`, "utf-8");
}
async function createLocalNonDottedFlow(tempDir: string, name: string) {
const flowDir = `${tempDir}/f/test/${name}__flow`;
await mkdir(flowDir, { recursive: true });
await writeFile(
`${flowDir}/a.ts`,
`export async function main() {\n return "Hello from flow ${name}";\n}`,
"utf-8"
);
await writeFile(
`${flowDir}/flow.yaml`,
`summary: "${name} flow"
description: "A flow for testing"
value:
modules:
- id: a
value:
type: rawscript
content: "!inline a.ts"
language: bun
input_transforms: {}
schema:
$schema: "https://json-schema.org/draft/2020-12/schema"
type: object
properties: {}
required: []
`,
"utf-8"
);
}
async function createLocalNonDottedApp(tempDir: string, name: string) {
const appDir = `${tempDir}/f/test/${name}__app`;
await mkdir(appDir, { recursive: true });
await writeFile(
`${appDir}/app.yaml`,
`summary: "${name} app"
value:
type: app
grid:
- id: button1
data:
type: buttoncomponent
componentInput:
type: runnable
runnable:
type: runnableByName
inlineScript:
content: |
export async function main() {
return "hello from app";
}
language: bun
hiddenInlineScripts: []
css: {}
norefreshbar: false
policy:
on_behalf_of: null
on_behalf_of_email: null
triggerables: {}
execution_mode: viewer
`,
"utf-8"
);
}
async function fileExists(filePath: string): Promise<boolean> {
try {
await stat(filePath);
return true;
} catch {
return false;
}
}
// =============================================================================
// Main test: processes scripts, flows, and apps together
// =============================================================================
test("generate-metadata: processes scripts, flows, and apps together", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "unified_all_test");
// Create one of each type
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
await createLocalApp(tempDir, "f/test", "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"unified_all_test"
);
expect(result.code).toEqual(0);
// Should find stale items
expect(result.stdout).toContain("Found");
expect(result.stdout).toContain("stale metadata");
});
});
// =============================================================================
// Flag tests
// =============================================================================
describe("generate-metadata flags", () => {
test("--includes filters to specific paths", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "includes_test");
// Create two scripts in different folders
await createLocalScript(tempDir, "f/included", "script_a");
await createLocalScript(tempDir, "f/excluded", "script_b");
// Run with --includes to only process f/included
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "-i", "f/included/**"],
tempDir,
"includes_test"
);
expect(result.code).toEqual(0);
// Should only mention the included script
const output = result.stdout + result.stderr;
expect(output).toContain("script_a");
expect(output).not.toContain("script_b");
});
});
test("--excludes filters out specific paths", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "excludes_test");
// Create two scripts
await createLocalScript(tempDir, "f/keep", "script_keep");
await createLocalScript(tempDir, "f/skip", "script_skip");
// Run with --excludes to skip f/skip
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "-e", "f/skip/**"],
tempDir,
"excludes_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("script_keep");
expect(output).not.toContain("script_skip");
});
});
test("--dry-run shows stale items without updating", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "dry_run_test");
await createLocalScript(tempDir, "f/test", "my_script");
// Run with --dry-run
const result = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"dry_run_test"
);
expect(result.code).toEqual(0);
// Should show stale items (Scripts section header)
expect(result.stdout).toContain("Scripts");
expect(result.stdout).toContain("my_script");
// Should NOT show "Done" (didn't actually update)
expect(result.stdout).not.toContain("Done");
// Run again without --dry-run to verify it would still be stale
const result2 = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"dry_run_test"
);
expect(result2.stdout).toContain("Scripts");
});
});
test("--lock-only only regenerates locks", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "lock_only_test");
await createLocalScript(tempDir, "f/test", "my_script");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--lock-only"],
tempDir,
"lock_only_test"
);
expect(result.code).toEqual(0);
});
});
test("--lock-only preserves non-dotted flow filenames", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "lock_only_non_dotted_test", true);
await createLocalNonDottedFlow(tempDir, "my_flow");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--lock-only"],
tempDir,
"lock_only_non_dotted_test"
);
expect(result.code).toEqual(0);
const flowDir = `${tempDir}/f/test/my_flow__flow`;
const flowYaml = await readFile(`${flowDir}/flow.yaml`, "utf-8");
expect(flowYaml).toContain("!inline a.ts");
expect(flowYaml).toContain("!inline a.lock");
expect(flowYaml).not.toContain(".inline_script.");
expect(await fileExists(`${flowDir}/a.lock`)).toEqual(true);
expect(await fileExists(`${flowDir}/a.inline_script.ts`)).toEqual(false);
expect(await fileExists(`${flowDir}/a.inline_script.lock`)).toEqual(false);
});
});
test("generate-metadata preserves non-dotted flow inline script filenames", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "full_gen_non_dotted_flow_test", true);
await createLocalNonDottedFlow(tempDir, "my_flow");
const result = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"full_gen_non_dotted_flow_test"
);
expect(result.code).toEqual(0);
const flowDir = `${tempDir}/f/test/my_flow__flow`;
const flowYaml = await readFile(`${flowDir}/flow.yaml`, "utf-8");
// Inline script references should use non-dotted naming
expect(flowYaml).toContain("!inline a.ts");
expect(flowYaml).toContain("!inline a.lock");
expect(flowYaml).not.toContain(".inline_script.");
expect(await fileExists(`${flowDir}/a.ts`)).toEqual(true);
expect(await fileExists(`${flowDir}/a.lock`)).toEqual(true);
expect(await fileExists(`${flowDir}/a.inline_script.ts`)).toEqual(false);
expect(await fileExists(`${flowDir}/a.inline_script.lock`)).toEqual(false);
});
});
test("generate-metadata uses non-dotted app inline script filenames", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "non_dotted_app_gen_test", true);
await createLocalNonDottedApp(tempDir, "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"non_dotted_app_gen_test"
);
expect(result.code).toEqual(0);
const appDir = `${tempDir}/f/test/my_app__app`;
const appYaml = await readFile(`${appDir}/app.yaml`, "utf-8");
// Inline script references should use non-dotted naming
expect(appYaml).not.toContain(".inline_script.");
// Verify no dotted inline script files were created
const { readdir: readdirAsync } = await import("node:fs/promises");
const files = await readdirAsync(appDir);
const dottedFiles = files.filter((f: string) => f.includes(".inline_script."));
expect(dottedFiles.length).toEqual(0);
});
});
test("--schema-only only processes scripts (skips flows and apps)", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "schema_only_test");
// Create one of each type
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
await createLocalApp(tempDir, "f/test", "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--schema-only"],
tempDir,
"schema_only_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should show "Checking scripts..." only
expect(output).toContain("Checking scripts...");
// Should find the script (Scripts section header)
expect(output).toContain("Scripts");
// Should NOT find flows or apps
expect(output).not.toContain("Flows");
expect(output).not.toContain("Apps");
});
});
test("--skip-scripts skips scripts", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_scripts_test");
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--skip-scripts"],
tempDir,
"skip_scripts_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should NOT contain script
expect(output).not.toContain("Scripts");
// Should contain flow
expect(output).toContain("Flows");
});
});
test("--skip-flows skips flows", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_flows_test");
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalFlow(tempDir, "f/test", "my_flow");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--skip-flows"],
tempDir,
"skip_flows_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should contain script
expect(output).toContain("Scripts");
// Should NOT contain flow
expect(output).not.toContain("Flows");
});
});
test("--skip-apps skips apps", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_apps_test");
await createLocalScript(tempDir, "f/test", "my_script");
await createLocalApp(tempDir, "f/test", "my_app");
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "--skip-apps"],
tempDir,
"skip_apps_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should contain script
expect(output).toContain("Scripts");
// Should NOT contain app
expect(output).not.toContain("Apps");
});
});
test("shows 'All metadata up-to-date' when nothing to update", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "uptodate_test");
// Create a script and run generate-metadata twice
await createLocalScript(tempDir, "f/test", "my_script");
// First run - generates metadata
await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"uptodate_test"
);
// Second run - should be up-to-date
const result = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"uptodate_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("up-to-date");
});
});
test("skipping all types shows warning", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "skip_all_test");
await createLocalScript(tempDir, "f/test", "my_script");
const result = await backend.runCLICommand(
["generate-metadata", "--skip-scripts", "--skip-flows", "--skip-apps"],
tempDir,
"skip_all_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("Nothing to check");
});
});
});
// =============================================================================
// Folder argument tests
// =============================================================================
describe("generate-metadata folder argument", () => {
test("filters to specific script folder", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_script_test");
// Create scripts in different folders
await createLocalScript(tempDir, "f/included", "script_a");
await createLocalScript(tempDir, "f/excluded", "script_b");
// Run with folder argument
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/included/script_a.ts"],
tempDir,
"folder_script_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("script_a");
expect(output).not.toContain("script_b");
});
});
test("filters to specific flow folder", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_flow_test");
// Create flows in different folders
await createLocalFlow(tempDir, "f/included", "flow_a");
await createLocalFlow(tempDir, "f/excluded", "flow_b");
// Run with folder argument (flow folder path - uses .flow suffix by default)
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/included/flow_a.flow"],
tempDir,
"folder_flow_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("flow_a");
expect(output).not.toContain("flow_b");
});
});
test("filters to specific app folder", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_app_test");
// Create apps in different folders
await createLocalApp(tempDir, "f/included", "app_a");
await createLocalApp(tempDir, "f/excluded", "app_b");
// Run with folder argument (app folder path - uses .app suffix by default)
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/included/app_a.app"],
tempDir,
"folder_app_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("app_a");
expect(output).not.toContain("app_b");
});
});
test("shows up-to-date when folder has no stale items", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "folder_uptodate_test");
await createLocalScript(tempDir, "f/test", "my_script");
// First run to generate metadata
await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"folder_uptodate_test"
);
// Second run with folder - should be up-to-date
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/test/my_script.ts"],
tempDir,
"folder_uptodate_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("up-to-date");
});
});
test("trailing slash is stripped (matches deprecated behavior)", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "trailing_slash_test");
await createLocalScript(tempDir, "f/test", "my_script");
// Run with trailing slash
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/test/my_script.ts/"],
tempDir,
"trailing_slash_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("my_script");
});
});
test("parent folder matches all children", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "parent_folder_test");
// Create scripts in nested folders
await createLocalScript(tempDir, "f/parent", "script_a");
await createLocalScript(tempDir, "f/parent/child", "script_b");
await createLocalScript(tempDir, "f/other", "script_c");
// Run with parent folder - should match both scripts in f/parent tree
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/parent"],
tempDir,
"parent_folder_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
expect(output).toContain("script_a");
expect(output).toContain("script_b");
expect(output).not.toContain("script_c");
});
});
test("non-existent folder shows up-to-date", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "nonexistent_folder_test");
await createLocalScript(tempDir, "f/exists", "my_script");
// Run with non-existent folder
const result = await backend.runCLICommand(
["generate-metadata", "--yes", "f/does_not_exist"],
tempDir,
"nonexistent_folder_test"
);
expect(result.code).toEqual(0);
expect(result.stdout).toContain("up-to-date");
});
});
});
// =============================================================================
// Scripts with modules
// =============================================================================
describe("generate-metadata with script modules", () => {
test("script with modules is detected as a single stale item", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "module_script_test");
// Create a script with module files
await createLocalScriptWithModules(tempDir, "f/test", "order_workflow", "bun", [
{ path: "helper.ts", content: 'export function validate() { return true; }\n' },
{ path: "utils.ts", content: 'export const VERSION = "1.0";\n' },
]);
const result = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"module_script_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// The main script should be listed as stale
expect(output).toContain("order_workflow");
// Module files should NOT appear as separate stale scripts (only within [changed modules: ...])
const lines = output.split("\n");
const staleLines = lines.filter((l: string) => l.includes("f/test/") || l.includes("f\\test\\"));
expect(staleLines.length).toBe(1);
expect(staleLines[0]).toContain("order_workflow");
});
});
test("module files are not treated as standalone scripts", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "module_not_standalone_test");
// Create a script with modules plus a regular script
await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
{ path: "helper.ts", content: 'export function greet() { return "hi"; }\n' },
]);
await createLocalScript(tempDir, "f/test", "standalone_script");
const result = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"module_not_standalone_test"
);
expect(result.code).toEqual(0);
const output = result.stdout + result.stderr;
// Should list both the main script and standalone script
expect(output).toContain("my_script");
expect(output).toContain("standalone_script");
// Should NOT list module helper as a separate entry
// Count occurrences of "Scripts" header — there should be exactly one
expect(output).toContain("Scripts");
});
});
test("script with modules generates metadata and becomes up-to-date", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "module_uptodate_test");
await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
{ path: "helper.ts", content: 'export function greet() { return "hi"; }\n' },
]);
// First run — should find stale items and generate metadata
const result1 = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"module_uptodate_test"
);
expect(result1.code).toEqual(0);
expect(result1.stdout).toContain("Done");
// Second run — should be up-to-date
const result2 = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"module_uptodate_test"
);
expect(result2.code).toEqual(0);
expect(result2.stdout).toContain("up-to-date");
});
});
test("modifying a module re-triggers stale detection", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "module_modify_test");
await createLocalScriptWithModules(tempDir, "f/test", "order_workflow", "bun", [
{ path: "helper.ts", content: 'export function greet() { return "hi"; }\n' },
{ path: "utils.ts", content: 'export const VERSION = "1.0";\n' },
]);
// First run — generate metadata
const result1 = await backend.runCLICommand(
["generate-metadata", "--yes"],
tempDir,
"module_modify_test"
);
expect(result1.code).toEqual(0);
// Second run — should be up-to-date
const result2 = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"module_modify_test"
);
expect(result2.code).toEqual(0);
const output2 = result2.stdout + result2.stderr;
expect(output2).not.toContain("order_workflow");
// Modify one module
await writeFile(
`${tempDir}/f/test/order_workflow__mod/helper.ts`,
'export function greet() { return "hello world"; }\n',
"utf-8"
);
// Third run — should detect the script as stale with the changed module
const result3 = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"module_modify_test"
);
expect(result3.code).toEqual(0);
const output3 = result3.stdout + result3.stderr;
expect(output3).toContain("order_workflow");
});
});
test("script with modules and lock files does not crash", async () => {
await withTestBackend(async (backend, tempDir) => {
await setupWorkspace(backend, tempDir, "module_with_locks_test");
// Create a script with modules that have lock files
await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
{
path: "helper.ts",
content: 'import lodash from "lodash";\nexport const x = lodash.identity(1);\n',
lock: "lodash@4.17.21\n",
},
]);
// Should not crash on lock files inside __mod/
const result = await backend.runCLICommand(
["generate-metadata", "--dry-run"],
tempDir,
"module_with_locks_test"
);
expect(result.code).toEqual(0);
// Should find the main script as stale, not crash on .lock files
expect(result.stdout).toContain("my_script");
});
});
});
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-utils-internal",
"version": "1.3.4",
"version": "1.3.5",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-utils-internal",
"version": "1.3.4",
"version": "1.3.5",
"license": "Apache 2.0",
"devDependencies": {
"@types/node": "^24.2.0",
+2 -2
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-utils-internal",
"version": "1.3.4",
"version": "1.3.5",
"description": "Internal utility functions for Windmill",
"main": "dist/cjs/index.js",
"module": "dist/esm/index.js",
@@ -31,4 +31,4 @@
"files": [
"dist/**/*"
]
}
}
@@ -150,8 +150,17 @@ export function extractInlineScripts(
*/
export function extractCurrentMapping(
modules: FlowModule[] | undefined,
mapping: Record<string, string> = {}
mapping: Record<string, string> = {},
failureModule?: FlowModule,
preprocessorModule?: FlowModule,
): Record<string, string> {
if (failureModule) {
extractCurrentMapping([failureModule], mapping);
}
if (preprocessorModule) {
extractCurrentMapping([preprocessorModule], mapping);
}
if (!modules || !Array.isArray(modules)) {
return mapping;
}
@@ -1,4 +1,11 @@
import { FlowModule, RawScript } from "../gen/types.gen.ts";
import { AiAgent, FlowModule, FlowValue, RawScript } from "../gen/types.gen.ts";
export type LocalScriptInfo = {
content: string;
language: RawScript["language"];
lock?: string;
tag?: string;
};
async function replaceRawscriptInline(
id: string,
@@ -118,4 +125,154 @@ export async function replaceInlineScripts(
}));
}
}));
}
}
/**
* Replaces PathScript ("script" type) modules with RawScript ("rawscript" type) using local file content.
* This is used during flow preview so that local script changes are tested instead of remote versions.
*
* @param modules - Array of flow modules to process
* @param scriptReader - Function that takes a script path and returns local content/language/lock, or undefined if not found locally
* @param logger - Logger for info/error messages
*/
export async function replacePathScriptsWithLocal(
modules: FlowModule[],
scriptReader: (scriptPath: string) => Promise<LocalScriptInfo | undefined>,
logger: {
info: (message: string) => void;
error: (message: string) => void;
} = {
info: () => {},
error: () => {},
}
): Promise<void> {
await Promise.all(modules.map(async (module) => {
if (!module.value) {
return;
}
if (module.value.type === "script") {
const scriptPath = module.value.path;
const localScript = await scriptReader(scriptPath);
if (localScript) {
const pathScript = module.value;
module.value = {
type: "rawscript",
content: localScript.content,
language: localScript.language,
lock: localScript.lock,
path: scriptPath,
input_transforms: pathScript.input_transforms,
tag: pathScript.tag_override ?? localScript.tag,
} satisfies RawScript;
}
} else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") {
await replacePathScriptsWithLocal(module.value.modules, scriptReader, logger);
} else if (module.value.type === "branchall") {
await Promise.all(module.value.branches.map(async (branch) => {
await replacePathScriptsWithLocal(branch.modules, scriptReader, logger);
}));
} else if (module.value.type === "branchone") {
await Promise.all(module.value.branches.map(async (branch) => {
await replacePathScriptsWithLocal(branch.modules, scriptReader, logger);
}));
await replacePathScriptsWithLocal(module.value.default, scriptReader, logger);
} else if (module.value.type === "aiagent") {
await Promise.all((module.value.tools ?? []).map(async (tool) => {
const toolValue = tool.value;
if (!toolValue || toolValue.tool_type !== "flowmodule" || toolValue.type !== "script") {
return;
}
const localScript = await scriptReader(toolValue.path);
if (localScript) {
(tool as AiAgent["tools"][number]).value = {
tool_type: "flowmodule",
type: "rawscript",
content: localScript.content,
language: localScript.language,
lock: localScript.lock,
path: toolValue.path,
input_transforms: toolValue.input_transforms,
tag: toolValue.tag_override ?? localScript.tag,
};
}
}));
}
}));
}
function collectPathScriptPathsFromModules(
modules: FlowModule[],
paths: Set<string>
): void {
for (const module of modules) {
if (!module.value) {
continue;
}
if (module.value.type === "script") {
paths.add(module.value.path);
} else if (
module.value.type === "forloopflow" ||
module.value.type === "whileloopflow"
) {
collectPathScriptPathsFromModules(module.value.modules, paths);
} else if (module.value.type === "branchall") {
for (const branch of module.value.branches) {
collectPathScriptPathsFromModules(branch.modules, paths);
}
} else if (module.value.type === "branchone") {
for (const branch of module.value.branches) {
collectPathScriptPathsFromModules(branch.modules, paths);
}
collectPathScriptPathsFromModules(module.value.default, paths);
} else if (module.value.type === "aiagent") {
for (const tool of module.value.tools ?? []) {
const toolValue = tool.value;
if (
toolValue &&
toolValue.tool_type === "flowmodule" &&
toolValue.type === "script"
) {
paths.add(toolValue.path);
}
}
}
}
}
/**
* Replaces all PathScript modules in a flow value (modules, failure_module, preprocessor_module)
* with RawScript using local file content.
*/
export async function replaceAllPathScriptsWithLocal(
flowValue: FlowValue,
scriptReader: (scriptPath: string) => Promise<LocalScriptInfo | undefined>,
logger: {
info: (message: string) => void;
error: (message: string) => void;
} = {
info: () => {},
error: () => {},
}
): Promise<void> {
await replacePathScriptsWithLocal(flowValue.modules, scriptReader, logger);
if (flowValue.failure_module) {
await replacePathScriptsWithLocal([flowValue.failure_module], scriptReader, logger);
}
if (flowValue.preprocessor_module) {
await replacePathScriptsWithLocal([flowValue.preprocessor_module], scriptReader, logger);
}
}
export function collectPathScriptPaths(flowValue: FlowValue): string[] {
const paths = new Set<string>();
collectPathScriptPathsFromModules(flowValue.modules, paths);
if (flowValue.failure_module) {
collectPathScriptPathsFromModules([flowValue.failure_module], paths);
}
if (flowValue.preprocessor_module) {
collectPathScriptPathsFromModules([flowValue.preprocessor_module], paths);
}
return [...paths];
}