mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-10-27 08:02:39 +00:00
Merge remote-tracking branch 'origin/main' into cli-onbehaflof
This commit is contained in:
@@ -12,6 +12,7 @@ const parserPackages = [
|
||||
"windmill-parser-wasm-yaml", "windmill-parser-wasm-csharp",
|
||||
"windmill-parser-wasm-nu", "windmill-parser-wasm-java",
|
||||
"windmill-parser-wasm-ruby",
|
||||
"windmill-parser-wasm-py-imports",
|
||||
];
|
||||
const parserExternals = parserPackages.flatMap(p => ["--external", p]);
|
||||
|
||||
|
||||
+5
-2
@@ -24,10 +24,11 @@
|
||||
"windmill-parser-wasm-nu": "*",
|
||||
"windmill-parser-wasm-php": "*",
|
||||
"windmill-parser-wasm-py": "*",
|
||||
"windmill-parser-wasm-py-imports": "*",
|
||||
"windmill-parser-wasm-regex": "*",
|
||||
"windmill-parser-wasm-ruby": "*",
|
||||
"windmill-parser-wasm-rust": "*",
|
||||
"windmill-parser-wasm-ts": "*",
|
||||
"windmill-parser-wasm-ts": "^1.659.1",
|
||||
"windmill-parser-wasm-yaml": "*",
|
||||
"windmill-yaml-validator": "1.1.1",
|
||||
"ws": "8.18.0",
|
||||
@@ -294,13 +295,15 @@
|
||||
|
||||
"windmill-parser-wasm-py": ["windmill-parser-wasm-py@1.628.3", "", {}, "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="],
|
||||
|
||||
"windmill-parser-wasm-py-imports": ["windmill-parser-wasm-py-imports@1.659.1", "", {}, "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="],
|
||||
|
||||
"windmill-parser-wasm-regex": ["windmill-parser-wasm-regex@1.639.0", "", {}, "sha512-qvYM4sYxB6M0xrqwBljS2fWqOMk6rp++60TRltJnzZDzVaWQrKjTGwNMmfepGAIWy1OGVKp0SCVERhe2P+O6tQ=="],
|
||||
|
||||
"windmill-parser-wasm-ruby": ["windmill-parser-wasm-ruby@1.526.1", "", {}, "sha512-rMBQA8s21wmL2kA5ztRs/ZgVA3ckxe9/NLjxl3iQPL0CX6DlvfaUH0O+AnhpXXDMyBs1Y1SZIhcnbnvsHZ3R8g=="],
|
||||
|
||||
"windmill-parser-wasm-rust": ["windmill-parser-wasm-rust@1.558.1", "", {}, "sha512-21S7lm1KF8zO1187rbq14hzPHII2RdM2+D44MoAh1F6VoaScj+Puq0z5B1O/hwn/95R/a9jBlL2D8jbkXtlD1A=="],
|
||||
|
||||
"windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.623.1", "", {}, "sha512-FBwi/zXxjhZcCvi04oFdNivazru1ynIqSbafHSArfaaBWesBO3nye9UO/WXUlWZm5a7BExbU+3R/eVJrGaornw=="],
|
||||
"windmill-parser-wasm-ts": ["windmill-parser-wasm-ts@1.659.1", "", {}, "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="],
|
||||
|
||||
"windmill-parser-wasm-yaml": ["windmill-parser-wasm-yaml@1.593.0", "", {}, "sha512-Gyx4aR2jsJYuDrD3mCNTmz7LWOQQXPw5yKNCC1xRgUOPfjsD/tINAFfsBLwVOSmlQQcFZO+wHm4KtDtXOcnGVw=="],
|
||||
|
||||
|
||||
Generated
+10
-4
@@ -25,10 +25,11 @@
|
||||
"windmill-parser-wasm-nu": "*",
|
||||
"windmill-parser-wasm-php": "*",
|
||||
"windmill-parser-wasm-py": "*",
|
||||
"windmill-parser-wasm-py-imports": "*",
|
||||
"windmill-parser-wasm-regex": "*",
|
||||
"windmill-parser-wasm-ruby": "*",
|
||||
"windmill-parser-wasm-rust": "*",
|
||||
"windmill-parser-wasm-ts": "*",
|
||||
"windmill-parser-wasm-ts": "^1.659.1",
|
||||
"windmill-parser-wasm-yaml": "*",
|
||||
"windmill-yaml-validator": "1.1.1",
|
||||
"ws": "8.18.0",
|
||||
@@ -1414,6 +1415,11 @@
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.628.3.tgz",
|
||||
"integrity": "sha512-TlluqknZpg8cZ+A3m6JFLPseY2PpKtDsxdj26fAnCUzKPtse8TxQR+n0dwC80rfW5TwdWSulvNGRDgcNuf7CTw=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-py-imports": {
|
||||
"version": "1.659.1",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-py-imports/-/windmill-parser-wasm-py-imports-1.659.1.tgz",
|
||||
"integrity": "sha512-nfnf04WBRf8f/mNIwdvggYOgz3erxrFGjKqULYBH+bKFMlKA6V7eB19m6CXOBkq9rjTp0ZFG+rgsR+Us7JEkyQ=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-regex": {
|
||||
"version": "1.639.0",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.639.0.tgz",
|
||||
@@ -1430,9 +1436,9 @@
|
||||
"integrity": "sha512-9yGLYZX2Hn9TdTqGY/5Fp50ftzgUsrfBkSK9vJkKJd5Amyg+yXLBGzd8pz6Org+4uxMenz/16wpsgijvo6uhhQ=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-ts": {
|
||||
"version": "1.647.1",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.647.1.tgz",
|
||||
"integrity": "sha512-64iSAUMU5W/WtePqE1vtDvglDqtkiZVndyieYBVDX0nl7UuovS+wPgH/P3TEoKbR+FwAPacki0CX3DsEzZ/Yxw=="
|
||||
"version": "1.659.1",
|
||||
"resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.659.1.tgz",
|
||||
"integrity": "sha512-EmXMzOmazC5r29UZh+1TVF9g/N2X51pqK11qDL6xWGeWTIIonhfOZ5nWdGvKQMDUR650fGxehImZzW2v9hNy+w=="
|
||||
},
|
||||
"node_modules/windmill-parser-wasm-yaml": {
|
||||
"version": "1.593.0",
|
||||
|
||||
+2
-1
@@ -32,10 +32,11 @@
|
||||
"windmill-parser-wasm-nu": "*",
|
||||
"windmill-parser-wasm-php": "*",
|
||||
"windmill-parser-wasm-py": "*",
|
||||
"windmill-parser-wasm-py-imports": "*",
|
||||
"windmill-parser-wasm-regex": "*",
|
||||
"windmill-parser-wasm-ruby": "*",
|
||||
"windmill-parser-wasm-rust": "*",
|
||||
"windmill-parser-wasm-ts": "*",
|
||||
"windmill-parser-wasm-ts": "^1.659.1",
|
||||
"windmill-parser-wasm-yaml": "*",
|
||||
"windmill-yaml-validator": "1.1.1",
|
||||
"ws": "8.18.0",
|
||||
|
||||
@@ -366,6 +366,9 @@ const command = new Command()
|
||||
"Default TypeScript runtime (bun or deno)"
|
||||
)
|
||||
.action(async (opts: any, appFolder: string | undefined) => {
|
||||
log.warn(
|
||||
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
|
||||
);
|
||||
const { generateLocksCommand } = await import("./app_metadata.ts");
|
||||
await generateLocksCommand(opts, appFolder);
|
||||
});
|
||||
|
||||
@@ -7,6 +7,7 @@ import { yamlParseFile } from "../../utils/yaml.ts";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { GlobalOptions } from "../../types.ts";
|
||||
import {
|
||||
readLockfile,
|
||||
checkifMetadataUptodate,
|
||||
blueColor,
|
||||
clearGlobalLock,
|
||||
@@ -41,6 +42,8 @@ import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts";
|
||||
import { resolveWorkspace } from "../../core/context.ts";
|
||||
import { requireLogin } from "../../core/auth.ts";
|
||||
import { getNonDottedPaths } from "../../utils/resource_folders.ts";
|
||||
import { extractRelativeImports } from "../../utils/relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts";
|
||||
|
||||
const TOP_HASH = "__app_hash";
|
||||
export const APP_BACKEND_FOLDER = "backend";
|
||||
@@ -93,7 +96,16 @@ async function generateAppHash(
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates locks for inline scripts in an app
|
||||
* Result of generating app locks, including which scripts were updated
|
||||
*/
|
||||
export interface AppLocksResult {
|
||||
path: string;
|
||||
updatedScripts: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates locks for inline scripts in an app.
|
||||
* Returns the path if dry-run, or AppLocksResult with updated scripts if actual update occurred.
|
||||
*/
|
||||
export async function generateAppLocksInternal(
|
||||
appFolder: string,
|
||||
@@ -104,8 +116,10 @@ export async function generateAppLocksInternal(
|
||||
defaultTs?: "bun" | "deno";
|
||||
},
|
||||
justUpdateMetadataLock?: boolean,
|
||||
noStaleMessage?: boolean
|
||||
): Promise<string | void> {
|
||||
noStaleMessage?: boolean,
|
||||
legacyBehaviour?: boolean,
|
||||
tree?: DoubleLinkedDependencyTree
|
||||
): Promise<string | AppLocksResult | void> {
|
||||
if (appFolder.endsWith(SEP)) {
|
||||
appFolder = appFolder.substring(0, appFolder.length - 1);
|
||||
}
|
||||
@@ -116,9 +130,6 @@ export async function generateAppLocksInternal(
|
||||
log.info(`Generating locks for app ${appFolder} at ${remote_path}`);
|
||||
}
|
||||
|
||||
const rawWorkspaceDependencies: Record<string, string> =
|
||||
await getRawWorkspaceDependencies();
|
||||
|
||||
// Read the app file first to filter workspace dependencies
|
||||
const appFilePath = path.join(
|
||||
appFolder,
|
||||
@@ -126,38 +137,83 @@ export async function generateAppLocksInternal(
|
||||
);
|
||||
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
|
||||
|
||||
// Filter workspace dependencies based on inline scripts' languages and annotations
|
||||
const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value;
|
||||
const filteredDeps = await filterWorkspaceDependenciesForApp(
|
||||
appValue,
|
||||
rawWorkspaceDependencies,
|
||||
appFolder
|
||||
);
|
||||
const folderNormalized = appFolder.replaceAll(SEP, "/");
|
||||
|
||||
let hashes = await generateAppHash(
|
||||
filteredDeps,
|
||||
appFolder,
|
||||
rawApp,
|
||||
opts.defaultTs
|
||||
);
|
||||
let filteredDeps: Record<string, string> = {};
|
||||
const conf = await readLockfile();
|
||||
|
||||
const conf = await import("../../utils/metadata.ts").then((m) =>
|
||||
m.readLockfile()
|
||||
);
|
||||
if (
|
||||
await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)
|
||||
) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
// New behaviour: tree-based dependency tracking
|
||||
if (!legacyBehaviour && tree) {
|
||||
if (dryRun) {
|
||||
const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
// For raw apps in new format, runnables are in separate files under backend/
|
||||
let treeAppValue = structuredClone(appValue);
|
||||
if (rawApp) {
|
||||
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER);
|
||||
const runnablesFromFiles = await loadRunnablesFromBackend(runnablesPath);
|
||||
if (Object.keys(runnablesFromFiles).length > 0) {
|
||||
treeAppValue = runnablesFromFiles;
|
||||
}
|
||||
}
|
||||
|
||||
// First pass: add inline scripts as separate nodes, then add app node importing them
|
||||
const inlineScriptPaths: string[] = [];
|
||||
await traverseAndProcessInlineScripts(treeAppValue, async (inlineScript, context) => {
|
||||
if (!inlineScript.content || !inlineScript.language) {
|
||||
return inlineScript;
|
||||
}
|
||||
|
||||
let content = inlineScript.content;
|
||||
// Resolve !inline references
|
||||
if (typeof content === "string" && content.startsWith("!inline ")) {
|
||||
const filePath = appFolder + SEP + content.replace("!inline ", "");
|
||||
try {
|
||||
content = await readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
return inlineScript;
|
||||
}
|
||||
}
|
||||
|
||||
const treePath = folderNormalized + "/" + context.path.join("/");
|
||||
const language = inlineScript.language as ScriptLanguage;
|
||||
const imports = await extractRelativeImports(content, treePath, language);
|
||||
await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, appFolder, false);
|
||||
inlineScriptPaths.push(treePath);
|
||||
|
||||
return inlineScript;
|
||||
});
|
||||
|
||||
await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "app", folderNormalized, appFolder, isDirectlyStale, rawApp);
|
||||
return;
|
||||
}
|
||||
// Second pass: get mismatched workspace deps from tree
|
||||
// TODO: pass raw workspace deps more precisely to every inline script lock generation call
|
||||
// (currently we pass the union of all mismatched deps filtered for the whole app)
|
||||
filteredDeps = await filterWorkspaceDependenciesForApp(appValue, tree.getMismatchedWorkspaceDeps(), appFolder);
|
||||
} else {
|
||||
// Legacy behaviour
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder);
|
||||
|
||||
const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
if (!isDirectlyStale) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
|
||||
if (Object.keys(filteredDeps).length > 0) {
|
||||
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Found workspace dependencies (${workspaceDependenciesLanguages
|
||||
@@ -167,7 +223,11 @@ export async function generateAppLocksInternal(
|
||||
);
|
||||
}
|
||||
|
||||
let updatedScripts: string[] = [];
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs);
|
||||
|
||||
const changedScripts = [];
|
||||
// Find hashes that do not correspond to previous hashes
|
||||
for (const [scriptPath, hash] of Object.entries(hashes)) {
|
||||
@@ -179,10 +239,18 @@ export async function generateAppLocksInternal(
|
||||
}
|
||||
}
|
||||
|
||||
if (changedScripts.length > 0) {
|
||||
log.info(
|
||||
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
|
||||
);
|
||||
// Get temp_script_refs from tree for relative import resolution
|
||||
const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized);
|
||||
|
||||
// In tree mode, the tree already verified this app is stale (possibly via dependency change).
|
||||
// Per-script hashes only detect content changes, not transitive dependency changes,
|
||||
// so we must regenerate locks for all inline scripts regardless.
|
||||
if (changedScripts.length > 0 || (tree && !legacyBehaviour)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
|
||||
);
|
||||
}
|
||||
|
||||
if (rawApp) {
|
||||
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER);
|
||||
@@ -199,13 +267,15 @@ export async function generateAppLocksInternal(
|
||||
replaceInlineScripts(runnables, runnablesPath + SEP, false);
|
||||
|
||||
// Update the app runnables with new locks (writes to separate files)
|
||||
await updateRawAppRunnables(
|
||||
updatedScripts = await updateRawAppRunnables(
|
||||
workspace,
|
||||
runnables,
|
||||
remote_path,
|
||||
appFolder,
|
||||
filteredDeps,
|
||||
opts.defaultTs
|
||||
opts.defaultTs,
|
||||
noStaleMessage,
|
||||
tempScriptRefs
|
||||
);
|
||||
// Note: updateRawAppRunnables now writes each runnable to its own file
|
||||
} else {
|
||||
@@ -215,14 +285,18 @@ export async function generateAppLocksInternal(
|
||||
replaceInlineScripts(normalAppFile.value, appFolder + SEP, false);
|
||||
|
||||
// Update the app value with new locks
|
||||
normalAppFile.value = await updateAppInlineScripts(
|
||||
const result = await updateAppInlineScripts(
|
||||
workspace,
|
||||
normalAppFile.value,
|
||||
remote_path,
|
||||
appFolder,
|
||||
filteredDeps,
|
||||
opts.defaultTs
|
||||
opts.defaultTs,
|
||||
noStaleMessage,
|
||||
tempScriptRefs
|
||||
);
|
||||
normalAppFile.value = result.value;
|
||||
updatedScripts = result.updatedScripts;
|
||||
|
||||
// Write the updated app file (only for normal apps, raw apps use separate files)
|
||||
writeIfChanged(
|
||||
@@ -230,23 +304,28 @@ export async function generateAppLocksInternal(
|
||||
yamlStringify(appFile as Record<string, any>, yamlOptions)
|
||||
);
|
||||
}
|
||||
} else {
|
||||
} else if (!noStaleMessage) {
|
||||
log.info(colors.gray(`No scripts changed in ${appFolder}`));
|
||||
}
|
||||
}
|
||||
|
||||
// Regenerate hashes after updates
|
||||
hashes = await generateAppHash(
|
||||
filteredDeps,
|
||||
// Non-legacy mode excludes workspace deps from hash (tracked via tree instead)
|
||||
const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps;
|
||||
const finalHashes = await generateAppHash(
|
||||
depsForHash,
|
||||
appFolder,
|
||||
rawApp,
|
||||
opts.defaultTs
|
||||
);
|
||||
await clearGlobalLock(appFolder);
|
||||
for (const [scriptPath, hash] of Object.entries(hashes)) {
|
||||
for (const [scriptPath, hash] of Object.entries(finalHashes)) {
|
||||
await updateMetadataGlobalLock(appFolder, hash, scriptPath);
|
||||
}
|
||||
log.info(colors.green(`App ${remote_path} lockfiles updated`));
|
||||
if (!noStaleMessage) {
|
||||
log.info(colors.green(`App ${remote_path} lockfiles updated`));
|
||||
}
|
||||
|
||||
return { path: remote_path, updatedScripts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -336,6 +415,7 @@ async function traverseAndProcessInlineScripts(
|
||||
* Updates locks for all runnables in a raw app, generating locks inline script by inline script.
|
||||
* Writes each runnable to its own YAML file in the backend folder (new format).
|
||||
* Also writes content and lock files to the runnables folder.
|
||||
* Returns the list of runnable IDs that had their locks updated.
|
||||
*/
|
||||
async function updateRawAppRunnables(
|
||||
workspace: Workspace,
|
||||
@@ -343,8 +423,11 @@ async function updateRawAppRunnables(
|
||||
remotePath: string,
|
||||
appFolder: string,
|
||||
rawDeps?: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" = "bun"
|
||||
): Promise<void> {
|
||||
defaultTs: "bun" | "deno" = "bun",
|
||||
noStaleMessage?: boolean,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string[]> {
|
||||
const updatedRunnables: string[] = [];
|
||||
const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER);
|
||||
|
||||
// Ensure runnables folder exists
|
||||
@@ -410,12 +493,11 @@ async function updateRawAppRunnables(
|
||||
continue;
|
||||
}
|
||||
|
||||
log.info(
|
||||
colors.gray(
|
||||
`Generating lock for runnable ${runnableId} (${language})
|
||||
}`
|
||||
)
|
||||
);
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(`Generating lock for runnable ${runnableId} (${language})`)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const lock = await generateInlineScriptLock(
|
||||
@@ -423,7 +505,8 @@ async function updateRawAppRunnables(
|
||||
content,
|
||||
language,
|
||||
`${remotePath}/${runnableId}`,
|
||||
rawDeps
|
||||
rawDeps,
|
||||
tempScriptRefs
|
||||
);
|
||||
|
||||
// Determine file extension for this language
|
||||
@@ -455,11 +538,15 @@ async function updateRawAppRunnables(
|
||||
// Write the runnable to its own YAML file
|
||||
writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable);
|
||||
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
updatedRunnables.push(runnableId);
|
||||
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch (error: any) {
|
||||
log.error(
|
||||
colors.red(
|
||||
@@ -470,11 +557,14 @@ async function updateRawAppRunnables(
|
||||
writeRunnableToBackend(runnablesFolder, runnableId, runnable);
|
||||
}
|
||||
}
|
||||
|
||||
return updatedRunnables;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates locks for all inline scripts in a normal app, similar to updateRawAppRunnables
|
||||
* but for the app.value structure instead of app.runnables
|
||||
* but for the app.value structure instead of app.runnables.
|
||||
* Returns a tuple of [updated app value, list of script names that were updated].
|
||||
*/
|
||||
async function updateAppInlineScripts(
|
||||
workspace: Workspace,
|
||||
@@ -482,9 +572,12 @@ async function updateAppInlineScripts(
|
||||
remotePath: string,
|
||||
appFolder: string,
|
||||
rawDeps?: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" = "bun"
|
||||
): Promise<any> {
|
||||
defaultTs: "bun" | "deno" = "bun",
|
||||
noStaleMessage?: boolean,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<{ value: any; updatedScripts: string[] }> {
|
||||
const pathAssigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() });
|
||||
const updatedScripts: string[] = [];
|
||||
|
||||
const processor: InlineScriptProcessor = async (inlineScript, context) => {
|
||||
const language = inlineScript.language as SupportedLanguage;
|
||||
@@ -514,20 +607,23 @@ async function updateAppInlineScripts(
|
||||
try {
|
||||
let lock: string | undefined;
|
||||
if (language !== "frontend") {
|
||||
log.info(
|
||||
colors.gray(
|
||||
`Generating lock for inline script "${scriptName}" at ${context.path.join(
|
||||
"."
|
||||
)} (${language})`
|
||||
)
|
||||
);
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(
|
||||
`Generating lock for inline script "${scriptName}" at ${context.path.join(
|
||||
"."
|
||||
)} (${language})`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
lock = await generateInlineScriptLock(
|
||||
workspace,
|
||||
content,
|
||||
language,
|
||||
scriptPath,
|
||||
rawDeps
|
||||
rawDeps,
|
||||
tempScriptRefs
|
||||
);
|
||||
}
|
||||
// Determine file extension for this language (following extractInlineScriptsForApps pattern)
|
||||
@@ -549,11 +645,18 @@ async function updateAppInlineScripts(
|
||||
const inlineLockRef =
|
||||
lock && lock !== "" ? `!inline ${basePath}lock` : "";
|
||||
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Track that this script was updated (only for non-frontend scripts that needed locks)
|
||||
if (language !== "frontend") {
|
||||
updatedScripts.push(scriptName);
|
||||
}
|
||||
|
||||
return {
|
||||
...inlineScript,
|
||||
@@ -573,7 +676,8 @@ async function updateAppInlineScripts(
|
||||
}
|
||||
};
|
||||
|
||||
return await traverseAndProcessInlineScripts(appValue, processor);
|
||||
const updatedValue = await traverseAndProcessInlineScripts(appValue, processor);
|
||||
return { value: updatedValue, updatedScripts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -584,7 +688,8 @@ async function generateInlineScriptLock(
|
||||
content: string,
|
||||
language: string,
|
||||
scriptPath: string,
|
||||
rawWorkspaceDependencies: Record<string, string> | undefined
|
||||
rawWorkspaceDependencies: Record<string, string> | undefined,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string> {
|
||||
// Filter workspace dependencies to only include those matching this script's language and annotations
|
||||
const filteredDeps = rawWorkspaceDependencies
|
||||
@@ -615,6 +720,9 @@ async function generateInlineScriptLock(
|
||||
? filteredDeps
|
||||
: null,
|
||||
entrypoint: scriptPath,
|
||||
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? { temp_script_refs: tempScriptRefs }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
@@ -767,7 +875,7 @@ export async function inferRunnableSchemaFromFile(
|
||||
}
|
||||
}
|
||||
|
||||
function getAppFolders(elems: Record<string, any>, extension: string) {
|
||||
export function getAppFolders(elems: Record<string, any>, extension: string) {
|
||||
return Object.keys(elems)
|
||||
.filter((p) => p.endsWith(SEP + extension))
|
||||
.map((p) => p.substring(0, p.length - (SEP + extension).length));
|
||||
|
||||
@@ -16,33 +16,34 @@ import { resolveWorkspace } from "../../core/context.ts";
|
||||
import {
|
||||
SyncOptions,
|
||||
mergeConfigWithConfigFile,
|
||||
readConfigFile,
|
||||
} from "../../core/conf.ts";
|
||||
import { exts, removeExtensionToPath } from "../script/script.ts";
|
||||
import { inferContentTypeFromFilePath } from "../../utils/script_common.ts";
|
||||
import { OpenFlow } from "../../../gen/types.gen.ts";
|
||||
import { FlowFile } from "../flow/flow.ts";
|
||||
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { replaceInlineScripts, replaceAllPathScriptsWithLocal } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { parseMetadataFile } from "../../utils/metadata.ts";
|
||||
import {
|
||||
getFolderSuffixWithSep,
|
||||
getMetadataFileName,
|
||||
extractFolderPath,
|
||||
} from "../../utils/resource_folders.ts";
|
||||
import { listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import { createPreviewLocalScriptReader } from "../../utils/local_path_scripts.ts";
|
||||
|
||||
const PORT = 3001;
|
||||
async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
log.info("Started dev mode");
|
||||
const conf = await readConfigFile();
|
||||
let currentLastEdit: LastEditScript | LastEditFlow | undefined = undefined;
|
||||
|
||||
const fsWatcher = watch(".", { recursive: true });
|
||||
const base = await realpath(".");
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
const ignore = await ignoreF(opts);
|
||||
const codebases = await listSyncCodebases(opts);
|
||||
|
||||
const changesTimeouts: Record<string, ReturnType<typeof setTimeout>> = {};
|
||||
function watchChanges() {
|
||||
@@ -56,7 +57,11 @@ async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
}
|
||||
changesTimeouts[key] = setTimeout(async () => {
|
||||
delete changesTimeouts[key];
|
||||
await loadPaths([filePath]);
|
||||
await loadPaths([filePath]).catch((error) => {
|
||||
log.error(
|
||||
`Failed to reload ${filePath}: ${error instanceof Error ? error.message : error}`
|
||||
);
|
||||
});
|
||||
}, 100);
|
||||
});
|
||||
fsWatcher.on("error", (err) => {
|
||||
@@ -94,6 +99,13 @@ async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
SEP,
|
||||
undefined,
|
||||
);
|
||||
// Replace PathScript modules with local file content so dev mode uses local versions
|
||||
const localScriptReader = createPreviewLocalScriptReader({
|
||||
exts,
|
||||
defaultTs: opts.defaultTs,
|
||||
codebases,
|
||||
});
|
||||
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
|
||||
currentLastEdit = {
|
||||
type: "flow",
|
||||
flow: localFlow,
|
||||
@@ -105,7 +117,7 @@ async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
const content = await readFile(cpath, "utf-8");
|
||||
const splitted = cpath.split(".");
|
||||
const wmPath = splitted[0];
|
||||
const lang = inferContentTypeFromFilePath(cpath, conf.defaultTs);
|
||||
const lang = inferContentTypeFromFilePath(cpath, opts.defaultTs);
|
||||
const typed =
|
||||
(await parseMetadataFile(
|
||||
removeExtensionToPath(cpath),
|
||||
|
||||
@@ -18,10 +18,21 @@ import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts";
|
||||
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
|
||||
import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts";
|
||||
import { Flow, OpenFlowWPath } from "../../../gen/types.gen.ts";
|
||||
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import {
|
||||
collectPathScriptPaths,
|
||||
replaceInlineScripts,
|
||||
replaceAllPathScriptsWithLocal,
|
||||
} from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { generateFlowLockInternal } from "./flow_metadata.ts";
|
||||
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
|
||||
import { resolvePermissionedAsRule } from "../../core/permissioned_as.ts";
|
||||
import { exts } from "../script/script.ts";
|
||||
import type { SyncCodebase } from "../../utils/codebase.ts";
|
||||
import { listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import {
|
||||
createPreviewLocalScriptReader,
|
||||
resolvePreviewLocalScriptState,
|
||||
} from "../../utils/local_path_scripts.ts";
|
||||
|
||||
export interface FlowFile {
|
||||
summary: string;
|
||||
@@ -31,6 +42,90 @@ export interface FlowFile {
|
||||
on_behalf_of_email?: string;
|
||||
}
|
||||
|
||||
function normalizeOptionalString(value: string | null | undefined): string | undefined {
|
||||
return typeof value === "string" && value.trim() === "" ? undefined : value ?? undefined;
|
||||
}
|
||||
|
||||
function normalizeComparableContent(value: string | undefined): string | undefined {
|
||||
return value?.replaceAll("\r\n", "\n").replace(/\n$/, "");
|
||||
}
|
||||
|
||||
async function findDivergedLocalPathScripts(
|
||||
workspaceId: string,
|
||||
scriptPaths: string[],
|
||||
opts: {
|
||||
exts: string[];
|
||||
defaultTs?: "bun" | "deno";
|
||||
codebases: SyncCodebase[];
|
||||
}
|
||||
): Promise<{ changed: string[]; missing: string[] }> {
|
||||
const changed: string[] = [];
|
||||
const missing: string[] = [];
|
||||
|
||||
for (const scriptPath of scriptPaths) {
|
||||
const localScript = await resolvePreviewLocalScriptState(scriptPath, opts);
|
||||
if (!localScript) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let remoteScript;
|
||||
try {
|
||||
remoteScript = await wmill.getScriptByPath({
|
||||
workspace: workspaceId,
|
||||
path: scriptPath,
|
||||
});
|
||||
} catch {
|
||||
missing.push(scriptPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
const remoteLock = normalizeOptionalString(remoteScript.lock);
|
||||
const diverged =
|
||||
normalizeComparableContent(localScript.content) !==
|
||||
normalizeComparableContent(remoteScript.content) ||
|
||||
localScript.language !== remoteScript.language ||
|
||||
(localScript.lock !== undefined &&
|
||||
normalizeComparableContent(localScript.lock) !==
|
||||
normalizeComparableContent(remoteLock)) ||
|
||||
localScript.tag !== normalizeOptionalString(remoteScript.tag) ||
|
||||
localScript.codebaseDigest !== normalizeOptionalString(remoteScript.codebase);
|
||||
|
||||
if (diverged) {
|
||||
changed.push(scriptPath);
|
||||
}
|
||||
}
|
||||
|
||||
return { changed, missing };
|
||||
}
|
||||
|
||||
function warnAboutLocalPathScriptDivergence(
|
||||
divergence: { changed: string[]; missing: string[] }
|
||||
): void {
|
||||
if (divergence.changed.length === 0 && divergence.missing.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const details: string[] = [];
|
||||
if (divergence.changed.length > 0) {
|
||||
details.push(
|
||||
`These workspace scripts differ from the deployed version:\n${divergence.changed
|
||||
.map((path) => `- ${path}`)
|
||||
.join("\n")}`
|
||||
);
|
||||
}
|
||||
if (divergence.missing.length > 0) {
|
||||
details.push(
|
||||
`These scripts do not exist in the workspace yet:\n${divergence.missing
|
||||
.map((path) => `- ${path}`)
|
||||
.join("\n")}`
|
||||
);
|
||||
}
|
||||
|
||||
log.warn(
|
||||
`Using local PathScript files for flow preview.\n${details.join("\n")}\nUse --remote to preview deployed workspace scripts instead.`
|
||||
);
|
||||
}
|
||||
|
||||
const alreadySynced: string[] = [];
|
||||
|
||||
export async function pushFlow(
|
||||
@@ -60,13 +155,20 @@ export async function pushFlow(
|
||||
}
|
||||
const localFlow = (await yamlParseFile(localPath + "flow.yaml")) as FlowFile;
|
||||
|
||||
const fileReader = async (path: string) => await readFile(localPath + path, "utf-8");
|
||||
await replaceInlineScripts(
|
||||
localFlow.value.modules,
|
||||
async (path: string) => await readFile(localPath + path, "utf-8"),
|
||||
fileReader,
|
||||
log,
|
||||
localPath,
|
||||
SEP
|
||||
);
|
||||
if (localFlow.value.failure_module) {
|
||||
await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, localPath, SEP);
|
||||
}
|
||||
if (localFlow.value.preprocessor_module) {
|
||||
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, localPath, SEP);
|
||||
}
|
||||
|
||||
// Build preserve flags for permissioned_as
|
||||
const preserveFields: Partial<OpenFlowWPath> = {};
|
||||
@@ -256,11 +358,17 @@ async function preview(
|
||||
opts: GlobalOptions & {
|
||||
data?: string;
|
||||
silent: boolean;
|
||||
},
|
||||
remote?: boolean;
|
||||
} & SyncOptions,
|
||||
flowPath: string
|
||||
) {
|
||||
const useLocalPathScripts = !opts.remote;
|
||||
if (useLocalPathScripts) {
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
}
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
const codebases = useLocalPathScripts ? listSyncCodebases(opts) : [];
|
||||
|
||||
// Normalize path - ensure it's a directory path to a .flow folder
|
||||
if (!flowPath.endsWith(".flow") && !flowPath.endsWith(".flow" + SEP)) {
|
||||
@@ -282,13 +390,45 @@ async function preview(
|
||||
const localFlow = (await yamlParseFile(flowPath + "flow.yaml")) as FlowFile;
|
||||
|
||||
// Replace inline scripts with their actual content
|
||||
const fileReader = async (path: string) => await readFile(flowPath + path, "utf-8");
|
||||
await replaceInlineScripts(
|
||||
localFlow.value.modules,
|
||||
async (path: string) => await readFile(flowPath + path, "utf-8"),
|
||||
fileReader,
|
||||
log,
|
||||
flowPath,
|
||||
SEP
|
||||
);
|
||||
if (localFlow.value.failure_module) {
|
||||
await replaceInlineScripts([localFlow.value.failure_module], fileReader, log, flowPath, SEP);
|
||||
}
|
||||
if (localFlow.value.preprocessor_module) {
|
||||
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, flowPath, SEP);
|
||||
}
|
||||
|
||||
if (useLocalPathScripts) {
|
||||
const scriptPaths = collectPathScriptPaths(localFlow.value);
|
||||
if (scriptPaths.length > 0) {
|
||||
const divergence = await findDivergedLocalPathScripts(
|
||||
workspace.workspaceId,
|
||||
scriptPaths,
|
||||
{
|
||||
exts,
|
||||
defaultTs: opts.defaultTs,
|
||||
codebases,
|
||||
}
|
||||
);
|
||||
if (!opts.silent) {
|
||||
warnAboutLocalPathScriptDivergence(divergence);
|
||||
}
|
||||
}
|
||||
|
||||
const localScriptReader = createPreviewLocalScriptReader({
|
||||
exts,
|
||||
defaultTs: opts.defaultTs,
|
||||
codebases,
|
||||
});
|
||||
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
|
||||
}
|
||||
|
||||
const input = opts.data ? await resolve(opts.data) : {};
|
||||
|
||||
@@ -324,12 +464,16 @@ async function preview(
|
||||
}
|
||||
}
|
||||
|
||||
async function generateLocks(
|
||||
export async function generateLocks(
|
||||
opts: GlobalOptions & {
|
||||
yes?: boolean;
|
||||
dryRun?: boolean;
|
||||
} & SyncOptions,
|
||||
folder: string | undefined
|
||||
) {
|
||||
log.warn(
|
||||
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
|
||||
);
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
@@ -374,6 +518,10 @@ async function generateLocks(
|
||||
}
|
||||
|
||||
if (hasAny) {
|
||||
if (opts.dryRun) {
|
||||
log.info(colors.gray("Dry run complete."));
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!opts.yes &&
|
||||
!(await Confirm.prompt({
|
||||
@@ -481,7 +629,7 @@ const command = new Command()
|
||||
.action(run as any)
|
||||
.command(
|
||||
"preview",
|
||||
"preview a local flow without deploying it. Runs the flow definition from local files."
|
||||
"preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default."
|
||||
)
|
||||
.arguments("<flow_path:string>")
|
||||
.option(
|
||||
@@ -492,6 +640,10 @@ const command = new Command()
|
||||
"-s --silent",
|
||||
"Do not output anything other then the final output. Useful for scripting."
|
||||
)
|
||||
.option(
|
||||
"--remote",
|
||||
"Use deployed workspace scripts for PathScript steps instead of local files."
|
||||
)
|
||||
.action(preview as any)
|
||||
.command(
|
||||
"generate-locks",
|
||||
@@ -499,6 +651,7 @@ const command = new Command()
|
||||
)
|
||||
.arguments("[flow:file]")
|
||||
.option("--yes", "Skip confirmation prompt")
|
||||
.option("--dry-run", "Perform a dry run without making changes")
|
||||
.option(
|
||||
"-i --includes <patterns:file[]>",
|
||||
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)"
|
||||
|
||||
@@ -19,6 +19,7 @@ import {
|
||||
} from "../../utils/metadata.ts";
|
||||
import { ScriptLanguage } from "../../utils/script_common.ts";
|
||||
import { extractInlineScripts as extractInlineScriptsForFlows } from "../../../windmill-utils-internal/src/inline-scripts/extractor.ts";
|
||||
import { newPathAssigner } from "../../../windmill-utils-internal/src/path-utils/path-assigner.ts";
|
||||
|
||||
import { generateHash, getHeaders, writeIfChanged } from "../../utils/utils.ts";
|
||||
import { exts } from "../script/script.ts";
|
||||
@@ -28,7 +29,9 @@ import { FlowFile } from "./flow.ts";
|
||||
import { FlowValue } from "../../../gen/types.gen.ts";
|
||||
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { workspaceDependenciesLanguages } from "../../utils/script_common.ts";
|
||||
import { extractNameFromFolder, getFolderSuffix } from "../../utils/resource_folders.ts";
|
||||
import { extractNameFromFolder, getFolderSuffix, getNonDottedPaths } from "../../utils/resource_folders.ts";
|
||||
import { extractRelativeImports } from "../../utils/relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts";
|
||||
|
||||
const TOP_HASH = "__flow_hash";
|
||||
async function generateFlowHash(
|
||||
@@ -50,6 +53,14 @@ async function generateFlowHash(
|
||||
}
|
||||
return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
|
||||
}
|
||||
/**
|
||||
* Result of generating flow locks, including which scripts were updated
|
||||
*/
|
||||
export interface FlowLocksResult {
|
||||
path: string;
|
||||
updatedScripts: string[];
|
||||
}
|
||||
|
||||
export async function generateFlowLockInternal(
|
||||
folder: string,
|
||||
dryRun: boolean,
|
||||
@@ -58,8 +69,10 @@ export async function generateFlowLockInternal(
|
||||
defaultTs?: "bun" | "deno";
|
||||
},
|
||||
justUpdateMetadataLock?: boolean,
|
||||
noStaleMessage?: boolean
|
||||
): Promise<string | void> {
|
||||
noStaleMessage?: boolean,
|
||||
legacyBehaviour?: boolean,
|
||||
tree?: DoubleLinkedDependencyTree
|
||||
): Promise<string | FlowLocksResult | void> {
|
||||
if (folder.endsWith(SEP)) {
|
||||
folder = folder.substring(0, folder.length - 1);
|
||||
}
|
||||
@@ -68,36 +81,70 @@ export async function generateFlowLockInternal(
|
||||
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
|
||||
}
|
||||
|
||||
// Always get out-of-sync workspace dependencies
|
||||
const rawWorkspaceDependencies: Record<string, string> =
|
||||
await getRawWorkspaceDependencies();
|
||||
|
||||
const flowValue = (await yamlParseFile(
|
||||
folder! + SEP + "flow.yaml"
|
||||
)) as FlowFile;
|
||||
|
||||
// Filter workspace dependencies based on inline scripts' languages and annotations
|
||||
const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
|
||||
|
||||
let hashes = await generateFlowHash(
|
||||
filteredDeps,
|
||||
folder,
|
||||
const folderNormalized = folder.replaceAll(SEP, "/");
|
||||
const inlineScriptsForTree = extractInlineScriptsForFlows(
|
||||
structuredClone(flowValue.value.modules),
|
||||
{},
|
||||
SEP,
|
||||
opts.defaultTs
|
||||
);
|
||||
).filter(s => !s.is_lock);
|
||||
|
||||
let filteredDeps: Record<string, string> = {};
|
||||
const conf = await readLockfile();
|
||||
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
|
||||
if (!legacyBehaviour && tree) {
|
||||
if (dryRun) {
|
||||
const inlineScriptPaths: string[] = [];
|
||||
for (const script of inlineScriptsForTree) {
|
||||
let content = script.content;
|
||||
if (content.startsWith("!inline ")) {
|
||||
const filePath = folder + SEP + content.replace("!inline ", "");
|
||||
try {
|
||||
content = await readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path));
|
||||
const language = script.language as ScriptLanguage;
|
||||
const imports = await extractRelativeImports(content, treePath, language);
|
||||
await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, folder, false);
|
||||
inlineScriptPaths.push(treePath);
|
||||
}
|
||||
|
||||
const hashes = await generateFlowHash({}, folder, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale);
|
||||
return;
|
||||
}
|
||||
// Second pass: get mismatched workspace deps from tree
|
||||
filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, tree.getMismatchedWorkspaceDeps(), folder);
|
||||
} else {
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
|
||||
|
||||
const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
if (!isDirectlyStale) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
|
||||
if (Object.keys(filteredDeps).length > 0) {
|
||||
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Found workspace dependencies (${workspaceDependenciesLanguages
|
||||
@@ -108,8 +155,25 @@ export async function generateFlowLockInternal(
|
||||
}
|
||||
|
||||
|
||||
let changedScripts: string[] = [];
|
||||
|
||||
// Build mapping from on-disk file names (hash keys like "a.py") to tree paths
|
||||
// (like "folder/a.inline_script"). The tree uses extractInlineScriptsForFlows without
|
||||
// a path assigner, so paths always have .inline_script suffix, but on-disk files
|
||||
// may not (non-dotted mode).
|
||||
const fileToTreePath = new Map<string, string>();
|
||||
for (const script of inlineScriptsForTree) {
|
||||
const c = script.content;
|
||||
if (c.startsWith("!inline ")) {
|
||||
const fileName = c.replace("!inline ", "");
|
||||
const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path));
|
||||
fileToTreePath.set(fileName, treePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
const changedScripts = [];
|
||||
const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs);
|
||||
|
||||
//find hashes that do not correspond to previous hashes
|
||||
for (const [path, hash] of Object.entries(hashes)) {
|
||||
if (path == TOP_HASH) {
|
||||
@@ -120,30 +184,61 @@ export async function generateFlowLockInternal(
|
||||
}
|
||||
}
|
||||
|
||||
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
|
||||
if (!noStaleMessage) {
|
||||
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
|
||||
}
|
||||
const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8");
|
||||
// In tree mode, use the tree's staleness info (which includes transitive dependency changes)
|
||||
// to determine which scripts need relocking, instead of only content-changed ones.
|
||||
const locksToRemove = (tree && !legacyBehaviour)
|
||||
? Object.keys(hashes).filter(k => {
|
||||
if (k === TOP_HASH) return false;
|
||||
const treePath = fileToTreePath.get(k)
|
||||
?? (folderNormalized + "/" + path.basename(k, path.extname(k)));
|
||||
return tree.isStale(treePath);
|
||||
})
|
||||
: changedScripts;
|
||||
await replaceInlineScripts(
|
||||
flowValue.value.modules,
|
||||
async (path: string) => await readFile(folder + SEP + path, "utf-8"),
|
||||
fileReader,
|
||||
log,
|
||||
folder + SEP!,
|
||||
SEP,
|
||||
changedScripts
|
||||
locksToRemove
|
||||
);
|
||||
if (flowValue.value.failure_module) {
|
||||
await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, locksToRemove);
|
||||
}
|
||||
if (flowValue.value.preprocessor_module) {
|
||||
await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, locksToRemove);
|
||||
}
|
||||
|
||||
//removeChangedLocks
|
||||
const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized);
|
||||
flowValue.value = await updateFlow(
|
||||
workspace,
|
||||
flowValue.value,
|
||||
remote_path,
|
||||
filteredDeps
|
||||
filteredDeps,
|
||||
tempScriptRefs
|
||||
);
|
||||
|
||||
const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", {
|
||||
skipInlineScriptSuffix: getNonDottedPaths(),
|
||||
});
|
||||
const inlineScripts = extractInlineScriptsForFlows(
|
||||
flowValue.value.modules,
|
||||
{},
|
||||
SEP,
|
||||
opts.defaultTs
|
||||
opts.defaultTs,
|
||||
lockAssigner
|
||||
);
|
||||
if (flowValue.value.failure_module) {
|
||||
inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.failure_module], {}, SEP, opts.defaultTs, lockAssigner));
|
||||
}
|
||||
if (flowValue.value.preprocessor_module) {
|
||||
inlineScripts.push(...extractInlineScriptsForFlows([flowValue.value.preprocessor_module], {}, SEP, opts.defaultTs, lockAssigner));
|
||||
}
|
||||
inlineScripts.forEach((s) => {
|
||||
writeIfChanged(process.cwd() + SEP + folder + SEP + s.path, s.content);
|
||||
});
|
||||
@@ -155,16 +250,36 @@ export async function generateFlowLockInternal(
|
||||
);
|
||||
}
|
||||
|
||||
hashes = await generateFlowHash(
|
||||
filteredDeps,
|
||||
// Non-legacy mode excludes workspace deps from hash (tracked via tree instead)
|
||||
const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps;
|
||||
const finalHashes = await generateFlowHash(
|
||||
depsForHash,
|
||||
folder,
|
||||
opts.defaultTs
|
||||
);
|
||||
await clearGlobalLock(folder);
|
||||
for (const [path, hash] of Object.entries(hashes)) {
|
||||
for (const [path, hash] of Object.entries(finalHashes)) {
|
||||
await updateMetadataGlobalLock(folder, hash, path);
|
||||
}
|
||||
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
|
||||
if (!noStaleMessage) {
|
||||
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
|
||||
}
|
||||
|
||||
// Return the list of updated scripts (extract just the filename from the path)
|
||||
// In tree mode, use the same staleness-aware list we used for lock removal
|
||||
const relocked = (tree && !legacyBehaviour)
|
||||
? Object.keys(finalHashes).filter(k => {
|
||||
if (k === TOP_HASH) return false;
|
||||
const treePath = fileToTreePath.get(k)
|
||||
?? (folderNormalized + "/" + path.basename(k, path.extname(k)));
|
||||
return tree.isStale(treePath);
|
||||
})
|
||||
: changedScripts;
|
||||
const updatedScripts = relocked.map(p => {
|
||||
const parts = p.split(SEP);
|
||||
return parts[parts.length - 1].replace(/\.[^.]+$/, ""); // Remove extension
|
||||
});
|
||||
return { path: remote_path, updatedScripts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -176,7 +291,15 @@ async function filterWorkspaceDependenciesForFlow(
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
folder: string
|
||||
): Promise<Record<string, string>> {
|
||||
const inlineScripts = extractInlineScriptsForFlows(structuredClone(flowValue.modules), {}, SEP, undefined);
|
||||
const clonedValue = structuredClone(flowValue);
|
||||
const depAssigner = newPathAssigner("bun");
|
||||
const inlineScripts = extractInlineScriptsForFlows(clonedValue.modules, {}, SEP, undefined, depAssigner);
|
||||
if (clonedValue.failure_module) {
|
||||
inlineScripts.push(...extractInlineScriptsForFlows([clonedValue.failure_module], {}, SEP, undefined, depAssigner));
|
||||
}
|
||||
if (clonedValue.preprocessor_module) {
|
||||
inlineScripts.push(...extractInlineScriptsForFlows([clonedValue.preprocessor_module], {}, SEP, undefined, depAssigner));
|
||||
}
|
||||
|
||||
// Filter out lock files and map to common interface
|
||||
const scripts = inlineScripts
|
||||
@@ -190,7 +313,8 @@ export async function updateFlow(
|
||||
workspace: Workspace,
|
||||
flow_value: FlowValue,
|
||||
remotePath: string,
|
||||
rawWorkspaceDependencies: Record<string, string>
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<FlowValue | undefined> {
|
||||
let rawResponse;
|
||||
|
||||
@@ -215,6 +339,9 @@ export async function updateFlow(
|
||||
path: remotePath,
|
||||
use_local_lockfiles: true,
|
||||
raw_workspace_dependencies: rawWorkspaceDependencies,
|
||||
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? { temp_script_refs: tempScriptRefs }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
@@ -233,6 +360,9 @@ export async function updateFlow(
|
||||
body: JSON.stringify({
|
||||
flow_value,
|
||||
path: remotePath,
|
||||
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? { temp_script_refs: tempScriptRefs }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -0,0 +1,446 @@
|
||||
import { Command } from "@cliffy/command";
|
||||
import { Confirm } from "@cliffy/prompt/confirm";
|
||||
import { colors } from "@cliffy/ansi/colors";
|
||||
import { sep as SEP } from "node:path";
|
||||
import { GlobalOptions } from "../../types.ts";
|
||||
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
|
||||
import { resolveWorkspace } from "../../core/context.ts";
|
||||
import { requireLogin } from "../../core/auth.ts";
|
||||
import * as log from "../../core/log.ts";
|
||||
import {
|
||||
generateScriptMetadataInternal,
|
||||
getRawWorkspaceDependencies,
|
||||
readLockfile,
|
||||
checkifMetadataUptodate,
|
||||
} from "../../utils/metadata.ts";
|
||||
import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts";
|
||||
import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts";
|
||||
import {
|
||||
elementsToMap,
|
||||
FSFSElement,
|
||||
ignoreF,
|
||||
} from "../sync/sync.ts";
|
||||
import { exts } from "../script/script.ts";
|
||||
import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts";
|
||||
import { listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import {
|
||||
DoubleLinkedDependencyTree,
|
||||
uploadScripts,
|
||||
ItemType,
|
||||
} from "../../utils/dependency_tree.ts";
|
||||
|
||||
interface StaleItem {
|
||||
type: ItemType;
|
||||
path: string;
|
||||
folder: string;
|
||||
isRawApp?: boolean;
|
||||
staleReason?: string;
|
||||
}
|
||||
|
||||
async function generateMetadata(
|
||||
opts: GlobalOptions & {
|
||||
yes?: boolean;
|
||||
lockOnly?: boolean;
|
||||
schemaOnly?: boolean;
|
||||
dryRun?: boolean;
|
||||
skipScripts?: boolean;
|
||||
skipFlows?: boolean;
|
||||
skipApps?: boolean;
|
||||
strictFolderBoundaries?: boolean;
|
||||
} & SyncOptions,
|
||||
folder?: string
|
||||
) {
|
||||
if (folder === "") {
|
||||
folder = undefined;
|
||||
}
|
||||
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(false);
|
||||
const codebases = await listSyncCodebases(opts);
|
||||
const ignore = await ignoreF(opts);
|
||||
|
||||
// --schema-only implies skipping flows and apps (they only have locks, no schemas)
|
||||
const skipScripts = opts.skipScripts ?? false;
|
||||
const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false;
|
||||
const skipApps = opts.skipApps ?? opts.schemaOnly ?? false;
|
||||
|
||||
const checking: string[] = [];
|
||||
if (!skipScripts) checking.push("scripts");
|
||||
if (!skipFlows) checking.push("flows");
|
||||
if (!skipApps) checking.push("apps");
|
||||
|
||||
if (checking.length === 0) {
|
||||
log.info(colors.yellow("Nothing to check (all types skipped)"));
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(`Checking ${checking.join(", ")}...`);
|
||||
|
||||
// Build dependency tree for relative import tracking
|
||||
const tree = new DoubleLinkedDependencyTree();
|
||||
tree.setWorkspaceDeps(rawWorkspaceDependencies);
|
||||
|
||||
// === Collect stale scripts ===
|
||||
if (!skipScripts) {
|
||||
// TODO: run elementsToMap only once but for all runnable types.
|
||||
const scriptElems = await elementsToMap(
|
||||
await FSFSElement(process.cwd(), codebases, false),
|
||||
(p, isD) => {
|
||||
return (
|
||||
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
|
||||
ignore(p, isD) ||
|
||||
isFolderResourcePathAnyFormat(p) ||
|
||||
(isScriptModulePath(p) && !isModuleEntryPoint(p))
|
||||
);
|
||||
},
|
||||
false,
|
||||
{}
|
||||
);
|
||||
|
||||
for (const e of Object.keys(scriptElems)) {
|
||||
await generateScriptMetadataInternal(
|
||||
e,
|
||||
workspace,
|
||||
opts,
|
||||
true, // dryRun - populate tree
|
||||
true, // noStaleMessage
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
false,
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// === Collect stale flows ===
|
||||
if (!skipFlows) {
|
||||
const flowElems = Object.keys(
|
||||
await elementsToMap(
|
||||
await FSFSElement(process.cwd(), [], true),
|
||||
(p, isD) => {
|
||||
return (
|
||||
ignore(p, isD) ||
|
||||
(!isD &&
|
||||
!p.endsWith(SEP + "flow.yaml") &&
|
||||
!p.endsWith(SEP + "flow.json"))
|
||||
);
|
||||
},
|
||||
false,
|
||||
{}
|
||||
)
|
||||
).map((x) => x.substring(0, x.lastIndexOf(SEP)));
|
||||
|
||||
for (const flowFolder of flowElems) {
|
||||
await generateFlowLockInternal(
|
||||
flowFolder,
|
||||
true, // dryRun - populate tree
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// === Collect stale apps ===
|
||||
if (!skipApps) {
|
||||
const elems = await elementsToMap(
|
||||
await FSFSElement(process.cwd(), [], true),
|
||||
(p, isD) => {
|
||||
return (
|
||||
ignore(p, isD) ||
|
||||
(!isD &&
|
||||
!p.endsWith(SEP + "raw_app.yaml") &&
|
||||
!p.endsWith(SEP + "app.yaml"))
|
||||
);
|
||||
},
|
||||
false,
|
||||
{}
|
||||
);
|
||||
|
||||
const rawAppFolders = getAppFolders(elems, "raw_app.yaml");
|
||||
const appFolders = getAppFolders(elems, "app.yaml");
|
||||
|
||||
for (const appFolder of rawAppFolders) {
|
||||
await generateAppLocksInternal(
|
||||
appFolder,
|
||||
true, // rawApp
|
||||
true, // dryRun - populate tree
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
}
|
||||
|
||||
for (const appFolder of appFolders) {
|
||||
await generateAppLocksInternal(
|
||||
appFolder,
|
||||
false, // rawApp
|
||||
true, // dryRun - populate tree
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// === Propagate staleness through imports ===
|
||||
tree.propagateStaleness();
|
||||
|
||||
// Upload stale scripts to temp storage so the backend can resolve relative imports.
|
||||
// If this fails (e.g. backend is older and doesn't have /raw_temp endpoints),
|
||||
// degrade gracefully: locks will be generated using deployed script content only.
|
||||
try {
|
||||
await uploadScripts(tree, workspace);
|
||||
} catch (e) {
|
||||
log.warn(colors.yellow(
|
||||
`Failed to upload scripts to temp storage (backend may be too old): ${e}. ` +
|
||||
`Locks will be generated using deployed script versions only — locally modified ` +
|
||||
`relative imports may not be reflected.`
|
||||
));
|
||||
}
|
||||
|
||||
// === Populate staleItems from tree ===
|
||||
const staleItems: StaleItem[] = [];
|
||||
const seenFolders = new Set<string>();
|
||||
|
||||
for (const p of tree.allPaths()) {
|
||||
const staleReason = tree.getStaleReason(p);
|
||||
if (!staleReason) continue;
|
||||
|
||||
const itemType = tree.getItemType(p)!;
|
||||
const itemFolder = tree.getFolder(p)!;
|
||||
|
||||
if (itemType === "dependencies") {
|
||||
staleItems.push({ type: itemType, path: p, folder: itemFolder, staleReason });
|
||||
} else if (itemType === "inline_script") {
|
||||
// Inline scripts are not listed separately — their parent flow/app is stale via propagation
|
||||
continue;
|
||||
} else if (itemType === "script") {
|
||||
const originalPath = tree.getOriginalPath(p)!;
|
||||
staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, staleReason });
|
||||
} else if (!seenFolders.has(itemFolder)) {
|
||||
// Flows/Apps: one entry per folder (dedupe multiple inline scripts)
|
||||
seenFolders.add(itemFolder);
|
||||
const originalPath = tree.getOriginalPath(p)!;
|
||||
staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, isRawApp: tree.getIsRawApp(p), staleReason });
|
||||
}
|
||||
}
|
||||
|
||||
// === Filter by folder if specified ===
|
||||
let filteredItems = staleItems;
|
||||
if (folder) {
|
||||
// Normalize to forward slashes (Windows users may use backslashes)
|
||||
folder = folder.replaceAll("\\", "/");
|
||||
// Strip trailing slash to match deprecated flow/app handler behavior
|
||||
if (folder.endsWith("/")) {
|
||||
folder = folder.substring(0, folder.length - 1);
|
||||
}
|
||||
// Strip file extension if user passed a specific file path (e.g. f/test/script.ts)
|
||||
const folderNoExt = folder.replace(/\.[^/.]+$/, "");
|
||||
// Check if an item is inside the specified folder
|
||||
const isInsideFolder = (item: StaleItem) => {
|
||||
const normalizedFolder = item.folder.replaceAll("\\", "/");
|
||||
const normalizedPath = item.path.replaceAll("\\", "/");
|
||||
return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/")
|
||||
|| normalizedPath === folder || normalizedPath === folderNoExt;
|
||||
};
|
||||
const isPathInFolder = (p: string) => p.startsWith(folder + "/") || p === folder || p === folderNoExt;
|
||||
// Check if a tree path or any of its transitive deps is inside the folder
|
||||
const touchesFolder = (treePath: string) => {
|
||||
if (isPathInFolder(treePath)) return true;
|
||||
let found = false;
|
||||
tree.traverseTransitive(treePath, (importPath) => {
|
||||
if (isPathInFolder(importPath)) {
|
||||
found = true;
|
||||
return true; // stop early
|
||||
}
|
||||
});
|
||||
return found;
|
||||
};
|
||||
|
||||
const isRelevant = (item: StaleItem) => {
|
||||
if (isInsideFolder(item)) return true;
|
||||
if (item.type === "dependencies") return true;
|
||||
const treePath = (item.type === "script"
|
||||
? item.path.replace(/\.[^/.]+$/, "")
|
||||
: item.folder).replaceAll("\\", "/");
|
||||
return touchesFolder(treePath);
|
||||
};
|
||||
|
||||
if (opts.strictFolderBoundaries) {
|
||||
// Strict mode: only items inside the folder
|
||||
filteredItems = staleItems.filter(isInsideFolder);
|
||||
|
||||
// Warn about stale items outside the folder that would be included by default
|
||||
const excludedStale = staleItems.filter((item) => !isInsideFolder(item) && isRelevant(item) && item.type !== "dependencies");
|
||||
for (const item of excludedStale) {
|
||||
const normalizedPath = item.path.replaceAll("\\", "/");
|
||||
log.warn(colors.yellow(
|
||||
`Warning: ${normalizedPath} depends on something inside "${folder}" but is outside it — skipped due to --strict-folder-boundaries. Next generate-metadata will not detect it as stale.`
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// Default: include items inside the folder and any stale importers that transitively depend on it
|
||||
filteredItems = staleItems.filter(isRelevant);
|
||||
}
|
||||
}
|
||||
|
||||
// === Show stale items and confirm ===
|
||||
if (filteredItems.length === 0) {
|
||||
log.info(colors.green("All metadata up-to-date"));
|
||||
return;
|
||||
}
|
||||
|
||||
// Group items by type for display
|
||||
const scripts = filteredItems.filter((i) => i.type === "script");
|
||||
const flows = filteredItems.filter((i) => i.type === "flow");
|
||||
const apps = filteredItems.filter((i) => i.type === "app");
|
||||
const deps = filteredItems.filter((i) => i.type === "dependencies");
|
||||
|
||||
log.info("");
|
||||
log.info(`Found ${colors.bold(String(filteredItems.length))} item(s) with stale metadata:`);
|
||||
|
||||
const printItems = (label: string, items: StaleItem[]) => {
|
||||
if (items.length === 0) return;
|
||||
log.info(` ${label} (${items.length}):`);
|
||||
for (const item of items) {
|
||||
const reason = item.staleReason ? colors.dim(colors.white(` — ${item.staleReason}`)) : "";
|
||||
log.info(` ~ ${item.path}` + reason);
|
||||
}
|
||||
};
|
||||
|
||||
printItems("Workspace dependencies", deps);
|
||||
printItems("Scripts", scripts);
|
||||
printItems("Flows", flows);
|
||||
printItems("Apps", apps);
|
||||
|
||||
if (opts.dryRun) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info("");
|
||||
|
||||
if (
|
||||
!opts.yes &&
|
||||
!(await Confirm.prompt({
|
||||
message: "Update metadata?",
|
||||
default: true,
|
||||
}))
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
log.info("");
|
||||
|
||||
// === Process all stale items with progress counter ===
|
||||
const mismatchedWorkspaceDeps = tree.getMismatchedWorkspaceDeps();
|
||||
const total = filteredItems.length - deps.length;
|
||||
const maxWidth = `[${total}/${total}]`.length;
|
||||
let current = 0;
|
||||
|
||||
const formatProgress = (n: number) => {
|
||||
return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " ")));
|
||||
};
|
||||
|
||||
// Process scripts
|
||||
for (const item of scripts) {
|
||||
current++;
|
||||
log.info(`${formatProgress(current)} script ${item.path}`);
|
||||
await generateScriptMetadataInternal(
|
||||
item.path, // originalPath with extension
|
||||
workspace,
|
||||
opts,
|
||||
false, // dryRun
|
||||
true, // noStaleMessage
|
||||
mismatchedWorkspaceDeps,
|
||||
codebases,
|
||||
false,
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
}
|
||||
|
||||
// Process flows
|
||||
for (const item of flows) {
|
||||
current++;
|
||||
const result = await generateFlowLockInternal(
|
||||
item.folder.replaceAll("/", SEP),
|
||||
false, // dryRun
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
const flowResult = result as FlowLocksResult | undefined;
|
||||
const scriptsInfo = flowResult?.updatedScripts?.length
|
||||
? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`))
|
||||
: "";
|
||||
log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`);
|
||||
}
|
||||
|
||||
// Process apps
|
||||
for (const item of apps) {
|
||||
current++;
|
||||
const result = await generateAppLocksInternal(
|
||||
item.folder.replaceAll("/", SEP),
|
||||
item.isRawApp!, // rawApp
|
||||
false, // dryRun
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
const appResult = result as AppLocksResult | undefined;
|
||||
const scriptsInfo = appResult?.updatedScripts?.length
|
||||
? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`))
|
||||
: "";
|
||||
log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`);
|
||||
}
|
||||
|
||||
// Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped)
|
||||
const allStaleDeps = staleItems.filter((i) => i.type === "dependencies");
|
||||
await tree.persistDepsHashes(allStaleDeps.map((d) => d.path));
|
||||
|
||||
log.info("");
|
||||
log.info(`Done. Updated ${colors.bold(String(total))} item(s).`);
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
.description("Generate metadata (locks, schemas) for all scripts, flows, and apps")
|
||||
.arguments("[folder:string]")
|
||||
.option("--yes", "Skip confirmation prompt")
|
||||
.option("--dry-run", "Show what would be updated without making changes")
|
||||
.option("--lock-only", "Re-generate only the lock files")
|
||||
.option("--schema-only", "Re-generate only script schemas (skips flows and apps)")
|
||||
.option("--skip-scripts", "Skip processing scripts")
|
||||
.option("--skip-flows", "Skip processing flows")
|
||||
.option("--skip-apps", "Skip processing apps")
|
||||
.option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)")
|
||||
.option(
|
||||
"-i --includes <patterns:file[]>",
|
||||
"Comma separated patterns to specify which files to include"
|
||||
)
|
||||
.option(
|
||||
"-e --excludes <patterns:file[]>",
|
||||
"Comma separated patterns to specify which files to exclude"
|
||||
)
|
||||
.action(generateMetadata as any);
|
||||
|
||||
export default command;
|
||||
@@ -252,6 +252,16 @@ async function initAction(opts: InitOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
// Read nonDottedPaths from config to specialize generated skills
|
||||
let nonDottedPaths = true; // default for new inits
|
||||
try {
|
||||
const { readConfigFile } = await import("../../core/conf.ts");
|
||||
const config = await readConfigFile();
|
||||
nonDottedPaths = config.nonDottedPaths ?? true;
|
||||
} catch {
|
||||
// If config can't be read, use default
|
||||
}
|
||||
|
||||
// Create guidance files (AGENTS.md, CLAUDE.md, and Claude skills)
|
||||
try {
|
||||
// Generate skills reference section for AGENTS.md
|
||||
@@ -290,6 +300,20 @@ async function initAction(opts: InitOptions) {
|
||||
|
||||
let skillContent = SKILL_CONTENT[skill.name];
|
||||
if (skillContent) {
|
||||
// Replace placeholders with actual suffixes based on nonDottedPaths
|
||||
if (nonDottedPaths) {
|
||||
skillContent = skillContent
|
||||
.replaceAll("{{FLOW_SUFFIX}}", "__flow")
|
||||
.replaceAll("{{APP_SUFFIX}}", "__app")
|
||||
.replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app")
|
||||
.replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`).");
|
||||
} else {
|
||||
skillContent = skillContent
|
||||
.replaceAll("{{FLOW_SUFFIX}}", ".flow")
|
||||
.replaceAll("{{APP_SUFFIX}}", ".app")
|
||||
.replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app")
|
||||
.replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`).");
|
||||
}
|
||||
// Check if this skill has schemas that need to be appended
|
||||
const schemaMappings = SCHEMA_MAPPINGS[skill.name];
|
||||
if (schemaMappings && schemaMappings.length > 0) {
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { GlobalOptions } from "../../types.ts";
|
||||
import { requireLogin } from "../../core/auth.ts";
|
||||
import { resolveWorkspace, validatePath } from "../../core/context.ts";
|
||||
import { readFile, writeFile, stat } from "node:fs/promises";
|
||||
import { readFile, writeFile, stat, mkdir } from "node:fs/promises";
|
||||
import { Buffer } from "node:buffer";
|
||||
import { colors } from "@cliffy/ansi/colors";
|
||||
import { Command } from "@cliffy/command";
|
||||
@@ -9,6 +9,7 @@ import { Confirm } from "@cliffy/prompt/confirm";
|
||||
import { Table } from "@cliffy/table";
|
||||
import * as log from "../../core/log.ts";
|
||||
import { sep as SEP } from "node:path";
|
||||
import * as path from "node:path";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { deepEqual } from "../../utils/utils.ts";
|
||||
import * as wmill from "../../../gen/services.gen.ts";
|
||||
@@ -51,7 +52,7 @@ import fs from "node:fs";
|
||||
import { createTarBlob, type TarEntry } from "../../utils/tar.ts";
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import { NewScript, Script } from "../../../gen/types.gen.ts";
|
||||
import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts";
|
||||
import type { PermissionedAsContext } from "../../core/permissioned_as.ts";
|
||||
import { resolvePermissionedAsRule } from "../../core/permissioned_as.ts";
|
||||
import {
|
||||
@@ -60,6 +61,12 @@ import {
|
||||
isFlowInlineScriptPath as isFlowInlineScriptPathInternal,
|
||||
isFlowPath,
|
||||
isAppPath,
|
||||
isScriptModulePath,
|
||||
buildModuleFolderPath,
|
||||
getModuleFolderSuffix,
|
||||
isModuleEntryPoint,
|
||||
getScriptBasePathFromModulePath,
|
||||
isRawAppPath,
|
||||
} from "../../utils/resource_folders.ts";
|
||||
|
||||
export interface ScriptFile {
|
||||
@@ -125,7 +132,7 @@ async function push(opts: PushOptions, filePath: string) {
|
||||
[],
|
||||
undefined,
|
||||
opts,
|
||||
await getRawWorkspaceDependencies(),
|
||||
await getRawWorkspaceDependencies(true),
|
||||
codebases
|
||||
);
|
||||
log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
|
||||
@@ -191,11 +198,17 @@ export async function handleScriptMetadata(
|
||||
opts: GlobalOptions,
|
||||
permissionedAsContext?: PermissionedAsContext
|
||||
): Promise<boolean> {
|
||||
if (
|
||||
path.endsWith(".script.json") ||
|
||||
// Flat layout: my_script.script.yaml
|
||||
const isFlatMeta = path.endsWith(".script.json") ||
|
||||
path.endsWith(".script.yaml") ||
|
||||
path.endsWith(".script.lock")
|
||||
) {
|
||||
path.endsWith(".script.lock");
|
||||
// Folder layout: my_script__mod/script.yaml
|
||||
const isFolderMeta = !isFlatMeta && isScriptModulePath(path) && (
|
||||
path.endsWith("/script.yaml") ||
|
||||
path.endsWith("/script.json") ||
|
||||
path.endsWith("/script.lock")
|
||||
);
|
||||
if (isFlatMeta || isFolderMeta) {
|
||||
const contentPath = await findContentFile(path);
|
||||
return handleFile(
|
||||
contentPath,
|
||||
@@ -230,10 +243,13 @@ export async function handleFile(
|
||||
codebases: SyncCodebase[],
|
||||
permissionedAsContext?: PermissionedAsContext
|
||||
): Promise<boolean> {
|
||||
// Detect module entry point: e.g., my_script__mod/script.ts
|
||||
const moduleEntryPoint = isModuleEntryPoint(path);
|
||||
if (
|
||||
!isAppInlineScriptPath(path) &&
|
||||
!isFlowInlineScriptPath(path) &&
|
||||
!isRawAppBackendPath(path) &&
|
||||
(!isScriptModulePath(path) || moduleEntryPoint) &&
|
||||
exts.some((exts) => path.endsWith(exts))
|
||||
) {
|
||||
if (alreadySynced.includes(path)) {
|
||||
@@ -242,9 +258,9 @@ export async function handleFile(
|
||||
log.debug(`Processing local script ${path}`);
|
||||
|
||||
alreadySynced.push(path);
|
||||
const remotePath = path
|
||||
.substring(0, path.indexOf("."))
|
||||
.replaceAll(SEP, "/");
|
||||
const remotePath = moduleEntryPoint
|
||||
? getScriptBasePathFromModulePath(path)!.replaceAll(SEP, "/")
|
||||
: path.substring(0, path.indexOf(".")).replaceAll(SEP, "/");
|
||||
|
||||
const language = inferContentTypeFromFilePath(path, opts?.defaultTs);
|
||||
|
||||
@@ -396,6 +412,13 @@ export async function handleFile(
|
||||
typed.codebase = await codebase.getDigest(forceTar);
|
||||
}
|
||||
|
||||
// Scan for modules: folder layout (entry point inside __mod/) or flat layout
|
||||
const scriptBasePath = moduleEntryPoint
|
||||
? getScriptBasePathFromModulePath(path)!
|
||||
: path.substring(0, path.indexOf("."));
|
||||
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
|
||||
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, moduleEntryPoint);
|
||||
|
||||
const requestBodyCommon: NewScript = {
|
||||
content,
|
||||
description: typed?.description ?? "",
|
||||
@@ -414,7 +437,6 @@ export async function handleFile(
|
||||
deployment_message: message,
|
||||
restart_unless_cancelled: typed?.restart_unless_cancelled,
|
||||
visible_to_runner_only: typed?.visible_to_runner_only,
|
||||
no_main_func: typed?.no_main_func,
|
||||
has_preprocessor: typed?.has_preprocessor,
|
||||
priority: typed?.priority,
|
||||
concurrency_key: typed?.concurrency_key,
|
||||
@@ -424,6 +446,7 @@ export async function handleFile(
|
||||
timeout: typed?.timeout,
|
||||
on_behalf_of_email: typed?.on_behalf_of_email,
|
||||
envs: typed?.envs,
|
||||
modules: modules,
|
||||
};
|
||||
|
||||
// Add preserve flags for permissioned_as
|
||||
@@ -474,7 +497,6 @@ export async function handleFile(
|
||||
Boolean(remote.restart_unless_cancelled) &&
|
||||
Boolean(typed.visible_to_runner_only) ==
|
||||
Boolean(remote.visible_to_runner_only) &&
|
||||
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
|
||||
Boolean(typed.has_preprocessor) ==
|
||||
Boolean(remote.has_preprocessor) &&
|
||||
typed.priority == Boolean(remote.priority) &&
|
||||
@@ -485,7 +507,8 @@ export async function handleFile(
|
||||
typed.debounce_delay_s == remote["debounce_delay_s"] &&
|
||||
typed.codebase == remote.codebase &&
|
||||
typed.on_behalf_of_email == remote.on_behalf_of_email &&
|
||||
deepEqual(typed.envs, remote.envs))
|
||||
deepEqual(typed.envs, remote.envs) &&
|
||||
deepEqual(modules ?? null, remote.modules ?? null))
|
||||
) {
|
||||
log.info(colors.green(`Script ${remotePath} is up to date`));
|
||||
return true;
|
||||
@@ -531,6 +554,135 @@ export async function handleFile(
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read module files from a __mod/ directory on disk.
|
||||
* Returns the modules record for the API, or undefined if no module folder exists.
|
||||
*/
|
||||
export async function readModulesFromDisk(
|
||||
moduleFolderPath: string,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
folderLayout: boolean = false,
|
||||
): Promise<Record<string, ScriptModule> | undefined> {
|
||||
if (!fs.existsSync(moduleFolderPath) || !fs.statSync(moduleFolderPath).isDirectory()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const modules: Record<string, ScriptModule> = {};
|
||||
|
||||
// In folder layout mode, skip the entry point files (script.*, script.yaml, etc.)
|
||||
const isEntryPointFile = (name: string, isTopLevel: boolean) => {
|
||||
if (!folderLayout || !isTopLevel) return false;
|
||||
return (
|
||||
name.startsWith("script.") ||
|
||||
name === "script.lock" ||
|
||||
name === "script.yaml" ||
|
||||
name === "script.json"
|
||||
);
|
||||
};
|
||||
|
||||
function readDir(dirPath: string, relPrefix: string) {
|
||||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
const isTopLevel = relPrefix === "";
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
readDir(fullPath, relPath);
|
||||
} else if (entry.isFile() && !entry.name.endsWith(".lock") && !isEntryPointFile(entry.name, isTopLevel)) {
|
||||
// Skip lock files — they're handled as the `lock` field on ScriptModule
|
||||
if (exts.some((ext) => entry.name.endsWith(ext))) {
|
||||
const content = fs.readFileSync(fullPath, "utf-8");
|
||||
const language = inferContentTypeFromFilePath(entry.name, defaultTs);
|
||||
|
||||
// Check for an accompanying lock file (helper.lock)
|
||||
const baseName = entry.name.replace(/\.[^.]+$/, '');
|
||||
const lockPath = path.join(dirPath, baseName + ".lock");
|
||||
let lock: string | undefined;
|
||||
if (fs.existsSync(lockPath)) {
|
||||
lock = fs.readFileSync(lockPath, "utf-8");
|
||||
}
|
||||
|
||||
modules[relPath] = {
|
||||
content,
|
||||
language: language as ScriptModule["language"],
|
||||
lock: lock ?? undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
readDir(moduleFolderPath, "");
|
||||
|
||||
if (Object.keys(modules).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
log.debug(`Found ${Object.keys(modules).length} module(s) in ${moduleFolderPath}`);
|
||||
return modules;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write module files to a __mod/ directory on disk during pull.
|
||||
*/
|
||||
export async function writeModulesToDisk(
|
||||
moduleFolderPath: string,
|
||||
modules: Record<string, ScriptModule>,
|
||||
defaultTs: "bun" | "deno" | undefined
|
||||
): Promise<void> {
|
||||
// Ensure the module folder exists
|
||||
fs.mkdirSync(moduleFolderPath, { recursive: true });
|
||||
|
||||
// Clean up stale module files that are no longer in the modules map
|
||||
const expectedFiles = new Set<string>();
|
||||
for (const [relPath, mod] of Object.entries(modules)) {
|
||||
expectedFiles.add(relPath);
|
||||
if (mod.lock) {
|
||||
expectedFiles.add(relPath.replace(/\.[^.]+$/, '') + ".lock");
|
||||
}
|
||||
}
|
||||
|
||||
function cleanDir(dirPath: string, relPrefix: string) {
|
||||
if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) return;
|
||||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
cleanDir(path.join(dirPath, entry.name), relPath);
|
||||
// Remove empty directories after cleaning
|
||||
try {
|
||||
const remaining = fs.readdirSync(path.join(dirPath, entry.name));
|
||||
if (remaining.length === 0) {
|
||||
fs.rmdirSync(path.join(dirPath, entry.name));
|
||||
}
|
||||
} catch {}
|
||||
} else if (!expectedFiles.has(relPath)) {
|
||||
fs.unlinkSync(path.join(dirPath, entry.name));
|
||||
}
|
||||
}
|
||||
}
|
||||
cleanDir(moduleFolderPath, "");
|
||||
|
||||
for (const [relPath, mod] of Object.entries(modules)) {
|
||||
const fullPath = path.join(moduleFolderPath, relPath);
|
||||
const dir = path.dirname(fullPath);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Write the module content
|
||||
fs.writeFileSync(fullPath, mod.content, "utf-8");
|
||||
|
||||
// Write the lock file if present
|
||||
if (mod.lock) {
|
||||
const baseName = relPath.replace(/\.[^.]+$/, '');
|
||||
const lockPath = path.join(moduleFolderPath, baseName + ".lock");
|
||||
const lockDir = path.dirname(lockPath);
|
||||
fs.mkdirSync(lockDir, { recursive: true });
|
||||
fs.writeFileSync(lockPath, mod.lock, "utf-8");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createScript(
|
||||
bundleContent: string | Blob | undefined,
|
||||
workspaceId: string,
|
||||
@@ -584,7 +736,12 @@ async function createScript(
|
||||
}
|
||||
|
||||
export async function findContentFile(filePath: string) {
|
||||
const candidates = filePath.endsWith("script.json")
|
||||
// Folder layout: __mod/script.yaml -> __mod/script.ts
|
||||
const isModuleFolderMeta =
|
||||
filePath.endsWith("/script.yaml") || filePath.endsWith("/script.json") || filePath.endsWith("/script.lock");
|
||||
const candidates = isModuleFolderMeta
|
||||
? exts.map((x) => filePath.replace(/\/script\.(yaml|json|lock)$/, "/script" + x))
|
||||
: filePath.endsWith("script.json")
|
||||
? exts.map((x) => filePath.replace(".script.json", x))
|
||||
: filePath.endsWith("script.lock")
|
||||
? exts.map((x) => filePath.replace(".script.lock", x))
|
||||
@@ -937,16 +1094,22 @@ async function get(opts: GlobalOptions & { json?: boolean }, path: string) {
|
||||
}
|
||||
}
|
||||
|
||||
const languageAliases: Record<string, ScriptLanguage> = {
|
||||
python: "python3",
|
||||
};
|
||||
|
||||
async function bootstrap(
|
||||
opts: GlobalOptions & { summary: string; description: string },
|
||||
scriptPath: string,
|
||||
language: ScriptLanguage
|
||||
language: ScriptLanguage | string
|
||||
) {
|
||||
if (!validatePath(scriptPath)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const scriptInitialCode = scriptBootstrapCode[language];
|
||||
const resolvedLanguage = (languageAliases[language] ?? language) as ScriptLanguage;
|
||||
|
||||
const scriptInitialCode = scriptBootstrapCode[resolvedLanguage];
|
||||
if (scriptInitialCode === undefined) {
|
||||
throw new Error("Language unknown");
|
||||
}
|
||||
@@ -954,7 +1117,7 @@ async function bootstrap(
|
||||
const config = await readConfigFile();
|
||||
|
||||
const extension = filePathExtensionFromContentType(
|
||||
language,
|
||||
resolvedLanguage,
|
||||
config.defaultTs
|
||||
);
|
||||
const scriptCodeFileFullPath = scriptPath + extension;
|
||||
@@ -986,6 +1149,9 @@ async function bootstrap(
|
||||
yamlOptions
|
||||
);
|
||||
|
||||
const parentDir = path.dirname(scriptCodeFileFullPath);
|
||||
await mkdir(parentDir, { recursive: true });
|
||||
|
||||
await writeFile(scriptCodeFileFullPath, scriptInitialCode, {
|
||||
flag: 'wx', encoding: 'utf-8',
|
||||
});
|
||||
@@ -1003,7 +1169,7 @@ export type GlobalDeps = Map<
|
||||
Record<string, string>
|
||||
>;
|
||||
|
||||
async function generateMetadata(
|
||||
export async function generateMetadata(
|
||||
opts: GlobalOptions & {
|
||||
lockOnly?: boolean;
|
||||
schemaOnly?: boolean;
|
||||
@@ -1011,6 +1177,9 @@ async function generateMetadata(
|
||||
} & SyncOptions,
|
||||
scriptPath: string | undefined
|
||||
) {
|
||||
log.warn(
|
||||
colors.yellow('This command is deprecated. Use "wmill generate-metadata" instead.')
|
||||
);
|
||||
log.info(
|
||||
"This command only works for workspace scripts, for flows inline scripts use `wmill flow generate-locks`"
|
||||
);
|
||||
@@ -1026,7 +1195,7 @@ async function generateMetadata(
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
const codebases = await listSyncCodebases(opts);
|
||||
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
if (scriptPath) {
|
||||
// read script metadata file
|
||||
await generateScriptMetadataInternal(
|
||||
@@ -1049,7 +1218,10 @@ async function generateMetadata(
|
||||
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
|
||||
ignore(p, isD) ||
|
||||
isFlowPath(p) ||
|
||||
isAppPath(p)
|
||||
isAppPath(p) ||
|
||||
isRawAppPath(p) ||
|
||||
// Skip module helper files; only entry points (script.{ext}) are processed
|
||||
(isScriptModulePath(p) && !isModuleEntryPoint(p))
|
||||
);
|
||||
},
|
||||
false,
|
||||
@@ -1138,6 +1310,13 @@ async function preview(
|
||||
const content = await readFile(filePath, "utf-8");
|
||||
const input = opts.data ? await resolve(opts.data) : {};
|
||||
|
||||
// Read modules from __mod/ folder if present
|
||||
const isFolderLayout = isModuleEntryPoint(filePath);
|
||||
const moduleFolderPath = isFolderLayout
|
||||
? path.dirname(filePath)
|
||||
: filePath.substring(0, filePath.indexOf(".")) + getModuleFolderSuffix();
|
||||
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, isFolderLayout);
|
||||
|
||||
// Check if this is a codebase script
|
||||
const codebase =
|
||||
language == "bun" ? findCodebase(filePath, codebases) : undefined;
|
||||
@@ -1297,6 +1476,7 @@ async function preview(
|
||||
path: filePath.substring(0, filePath.indexOf(".")).replaceAll(SEP, "/"),
|
||||
args: input,
|
||||
language: language as any,
|
||||
modules: modules ?? undefined,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
+282
-17
@@ -31,6 +31,7 @@ import {
|
||||
findResourceFile,
|
||||
handleScriptMetadata,
|
||||
removeExtensionToPath,
|
||||
filePathExtensionFromContentType,
|
||||
} from "../script/script.ts";
|
||||
|
||||
import { handleFile } from "../script/script.ts";
|
||||
@@ -69,7 +70,7 @@ import {
|
||||
readLockfile,
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
} from "../../utils/metadata.ts";
|
||||
import { OpenFlow, NativeServiceName } from "../../../gen/types.gen.ts";
|
||||
import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts";
|
||||
import { pushResource } from "../resource/resource.ts";
|
||||
import {
|
||||
newPathAssigner,
|
||||
@@ -103,6 +104,10 @@ import {
|
||||
getFolderSuffix,
|
||||
getFolderSuffixWithSep,
|
||||
getNonDottedPaths,
|
||||
isScriptModulePath,
|
||||
getModuleFolderSuffix,
|
||||
isModuleEntryPoint,
|
||||
getScriptBasePathFromModulePath,
|
||||
} from "../../utils/resource_folders.ts";
|
||||
|
||||
// Merge CLI options with effective settings, preserving CLI flags as overrides
|
||||
@@ -544,6 +549,29 @@ function ZipFSElement(
|
||||
ignoreCodebaseChanges: boolean,
|
||||
stripOnBehalfOf: boolean,
|
||||
): DynFSElement {
|
||||
// Pre-scan: find zip base paths of scripts that have modules.
|
||||
// These scripts use the folder layout: {basePath}__mod/script.{ext}
|
||||
let _moduleScriptPaths: Set<string> | null = null;
|
||||
async function getModuleScriptPaths(): Promise<Set<string>> {
|
||||
if (_moduleScriptPaths === null) {
|
||||
_moduleScriptPaths = new Set();
|
||||
for (const filename in zip.files) {
|
||||
if (filename.endsWith(".script.json") && !zip.files[filename].dir) {
|
||||
try {
|
||||
const content = await zip.files[filename].async("text");
|
||||
const parsed = JSON.parse(content);
|
||||
if (parsed.modules && Object.keys(parsed.modules).length > 0) {
|
||||
_moduleScriptPaths.add(
|
||||
filename.slice(0, -".script.json".length)
|
||||
);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
return _moduleScriptPaths;
|
||||
}
|
||||
|
||||
async function _internal_file(
|
||||
p: string,
|
||||
f: JSZip.JSZipObject,
|
||||
@@ -585,7 +613,22 @@ function ZipFSElement(
|
||||
}
|
||||
}
|
||||
|
||||
const finalPath = transformPath();
|
||||
let finalPath = transformPath();
|
||||
|
||||
// Redirect content files for scripts with modules into __mod/ folder
|
||||
if (kind == "other" && exts.some((ext) => p.endsWith(ext))) {
|
||||
const normalizedP = p.replace(/^\.[\\/]/, "");
|
||||
const moduleScripts = await getModuleScriptPaths();
|
||||
for (const basePath of moduleScripts) {
|
||||
if (normalizedP.startsWith(basePath + ".")) {
|
||||
const ext = normalizedP.slice(basePath.length); // e.g., ".ts", ".py"
|
||||
const dir = path.dirname(finalPath);
|
||||
const base = path.basename(basePath);
|
||||
finalPath = path.join(dir, base + getModuleFolderSuffix(), "script" + ext);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const r = [
|
||||
{
|
||||
@@ -602,14 +645,35 @@ function ZipFSElement(
|
||||
}
|
||||
let inlineScripts;
|
||||
try {
|
||||
const assigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() });
|
||||
inlineScripts = extractInlineScriptsForFlows(
|
||||
flow.value.modules as any,
|
||||
{},
|
||||
SEP,
|
||||
defaultTs,
|
||||
undefined, // pathAssigner - let it create one
|
||||
assigner,
|
||||
{ skipInlineScriptSuffix: getNonDottedPaths() },
|
||||
);
|
||||
if (flow.value.failure_module) {
|
||||
inlineScripts.push(...extractInlineScriptsForFlows(
|
||||
[flow.value.failure_module],
|
||||
{},
|
||||
SEP,
|
||||
defaultTs,
|
||||
assigner,
|
||||
{ skipInlineScriptSuffix: getNonDottedPaths() },
|
||||
));
|
||||
}
|
||||
if (flow.value.preprocessor_module) {
|
||||
inlineScripts.push(...extractInlineScriptsForFlows(
|
||||
[flow.value.preprocessor_module],
|
||||
{},
|
||||
SEP,
|
||||
defaultTs,
|
||||
assigner,
|
||||
{ skipInlineScriptSuffix: getNonDottedPaths() },
|
||||
));
|
||||
}
|
||||
} catch (error) {
|
||||
log.error(
|
||||
`Failed to extract inline scripts for flow at path: ${p}`,
|
||||
@@ -886,15 +950,23 @@ function ZipFSElement(
|
||||
log.error(`Failed to parse script.yaml at path: ${p}`);
|
||||
throw error;
|
||||
}
|
||||
const hasModules = parsed["modules"] && Object.keys(parsed["modules"]).length > 0;
|
||||
if (
|
||||
parsed["lock"] &&
|
||||
parsed["lock"] != "" &&
|
||||
parsed["codebase"] == undefined
|
||||
) {
|
||||
parsed["lock"] =
|
||||
"!inline " +
|
||||
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
|
||||
".lock";
|
||||
if (hasModules) {
|
||||
// Lock lives inside __mod/ folder as script.lock
|
||||
const scriptBase = removeSuffix(removeSuffix(p.replaceAll(SEP, "/"), ".json"), ".script");
|
||||
parsed["lock"] =
|
||||
"!inline " + scriptBase + getModuleFolderSuffix() + "/script.lock";
|
||||
} else {
|
||||
parsed["lock"] =
|
||||
"!inline " +
|
||||
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
|
||||
".lock";
|
||||
}
|
||||
} else if (parsed["lock"] == "") {
|
||||
parsed["lock"] = "";
|
||||
} else {
|
||||
@@ -906,6 +978,8 @@ function ZipFSElement(
|
||||
if (stripOnBehalfOf) {
|
||||
delete parsed["on_behalf_of_email"];
|
||||
}
|
||||
// Modules are stored as files in __mod/ folder, not in metadata
|
||||
delete parsed["modules"];
|
||||
return useYaml
|
||||
? yamlStringify(parsed, yamlOptions)
|
||||
: JSON.stringify(parsed, null, 2);
|
||||
@@ -965,16 +1039,71 @@ function ZipFSElement(
|
||||
throw error;
|
||||
}
|
||||
const lock = parsed["lock"];
|
||||
const scriptModules: Record<string, ScriptModule> | undefined = parsed["modules"];
|
||||
const hasModules = scriptModules && Object.keys(scriptModules).length > 0;
|
||||
|
||||
// Compute base path and module folder
|
||||
const metaExt = useYaml ? ".yaml" : ".json";
|
||||
const scriptBasePath = removeSuffix(
|
||||
removeSuffix(finalPath, metaExt),
|
||||
".script"
|
||||
);
|
||||
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
|
||||
|
||||
if (hasModules) {
|
||||
// Redirect metadata into __mod/script.yaml
|
||||
r[0].path = path.join(moduleFolderPath, "script" + metaExt);
|
||||
}
|
||||
|
||||
if (lock && lock != "") {
|
||||
r.push({
|
||||
isDirectory: false,
|
||||
path: removeSuffix(finalPath, ".json") + ".lock",
|
||||
path: hasModules
|
||||
? path.join(moduleFolderPath, "script.lock")
|
||||
: removeSuffix(finalPath, metaExt) + ".lock",
|
||||
async *getChildren() {},
|
||||
async getContentText() {
|
||||
return lock;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Extract script modules into __mod/ folder
|
||||
if (hasModules) {
|
||||
r.push({
|
||||
isDirectory: true,
|
||||
path: moduleFolderPath,
|
||||
async *getChildren() {
|
||||
for (const [relPath, mod] of Object.entries(scriptModules!)) {
|
||||
// Yield the module content file
|
||||
yield {
|
||||
isDirectory: false,
|
||||
path: path.join(moduleFolderPath, relPath),
|
||||
async *getChildren() {},
|
||||
async getContentText() {
|
||||
return mod.content;
|
||||
},
|
||||
};
|
||||
|
||||
// Yield the module lock file if present
|
||||
if (mod.lock) {
|
||||
const baseName = relPath.replace(/\.[^.]+$/, '');
|
||||
yield {
|
||||
isDirectory: false,
|
||||
path: path.join(moduleFolderPath, baseName + ".lock"),
|
||||
async *getChildren() {},
|
||||
async getContentText() {
|
||||
return mod.lock!;
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
async getContentText() {
|
||||
throw new Error("Cannot get content of directory");
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
if (kind == "resource") {
|
||||
const content = await f.async("text");
|
||||
@@ -1150,6 +1279,12 @@ export async function elementsToMap(
|
||||
continue;
|
||||
}
|
||||
const path = entry.path;
|
||||
// Include module files in the map so they're compared for changes,
|
||||
// but they're pushed as part of their parent script via handleFile
|
||||
if (isScriptModulePath(path)) {
|
||||
map[path] = await entry.getContentText();
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!isFileResource(path) &&
|
||||
!isFilesetResource(path) &&
|
||||
@@ -1596,6 +1731,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
|
||||
);
|
||||
}
|
||||
|
||||
// Files inside __mod/ folders are script module files — always valid wmill files
|
||||
if (isScriptModulePath(p)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const typ = getTypeStrFromPath(p);
|
||||
if (
|
||||
@@ -1740,6 +1880,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
|
||||
if (!tracker.rawApps.includes(folder)) {
|
||||
tracker.rawApps.push(folder);
|
||||
}
|
||||
} else if (isScriptModulePath(p)) {
|
||||
if (isModuleEntryPoint(p)) {
|
||||
// Entry point (e.g. __mod/script.ts) IS the parent script content file
|
||||
if (!tracker.scripts.includes(p)) {
|
||||
tracker.scripts.push(p);
|
||||
}
|
||||
} else {
|
||||
// Module file changed — find the parent script content file
|
||||
const moduleSuffix = getModuleFolderSuffix() + "/";
|
||||
const idx = p.indexOf(moduleSuffix);
|
||||
if (idx !== -1) {
|
||||
const scriptBasePath = p.substring(0, idx);
|
||||
// Try folder layout first: __mod/script.{ext}
|
||||
try {
|
||||
const contentPath = await findContentFile(scriptBasePath + getModuleFolderSuffix() + "/script.yaml");
|
||||
if (contentPath && !tracker.scripts.includes(contentPath)) {
|
||||
tracker.scripts.push(contentPath);
|
||||
}
|
||||
} catch {
|
||||
// Fall back to flat layout: scriptBasePath.script.yaml
|
||||
try {
|
||||
const contentPath = await findContentFile(scriptBasePath + ".script.yaml");
|
||||
if (contentPath && !tracker.scripts.includes(contentPath)) {
|
||||
tracker.scripts.push(contentPath);
|
||||
}
|
||||
} catch {
|
||||
// ignore — content file not found
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (!tracker.scripts.includes(p)) {
|
||||
tracker.scripts.push(p);
|
||||
@@ -1772,6 +1943,61 @@ async function buildTracker(changes: Change[]) {
|
||||
return tracker;
|
||||
}
|
||||
|
||||
/**
|
||||
* When a module file changes, find and push the parent script.
|
||||
* The parent script's handleFile will read the __mod/ folder and include all modules.
|
||||
*/
|
||||
async function pushParentScriptForModule(
|
||||
modulePath: string,
|
||||
workspace: Workspace,
|
||||
alreadySynced: string[],
|
||||
message: string | undefined,
|
||||
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
codebases: SyncCodebase[],
|
||||
): Promise<void> {
|
||||
const moduleSuffix = getModuleFolderSuffix() + "/";
|
||||
const idx = modulePath.indexOf(moduleSuffix);
|
||||
if (idx === -1) return;
|
||||
const scriptBasePath = modulePath.substring(0, idx);
|
||||
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
|
||||
|
||||
// Try folder layout first: look for script.{ext} inside __mod/
|
||||
try {
|
||||
const entryPoint = await findContentFile(moduleFolderPath + "/script.yaml");
|
||||
if (entryPoint) {
|
||||
await handleFile(
|
||||
entryPoint,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
return;
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Fall back to flat layout: look for content file alongside __mod/
|
||||
try {
|
||||
const contentPath = await findContentFile(scriptBasePath + ".script.yaml");
|
||||
if (contentPath) {
|
||||
await handleFile(
|
||||
contentPath,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
log.debug(`Could not find parent script for module: ${modulePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function pull(
|
||||
opts: GlobalOptions &
|
||||
SyncOptions & { repository?: string; promotion?: string; branch?: string },
|
||||
@@ -2072,7 +2298,7 @@ export async function pull(
|
||||
|
||||
const tracker: ChangeTracker = await buildTracker(changes);
|
||||
const rawWorkspaceDependencies: Record<string, string> =
|
||||
await getRawWorkspaceDependencies();
|
||||
await getRawWorkspaceDependencies(true);
|
||||
|
||||
for (const change of tracker.scripts) {
|
||||
await generateScriptMetadataInternal(
|
||||
@@ -2404,7 +2630,7 @@ export async function push(
|
||||
false, // els1 (local) is not the remote source
|
||||
);
|
||||
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
|
||||
const tracker: ChangeTracker = await buildTracker(changes);
|
||||
|
||||
@@ -2450,7 +2676,7 @@ export async function push(
|
||||
true,
|
||||
);
|
||||
if (stale) {
|
||||
staleFlows.push(stale);
|
||||
staleFlows.push(stale as string);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2475,7 +2701,7 @@ export async function push(
|
||||
true,
|
||||
);
|
||||
if (stale) {
|
||||
staleApps.push(stale);
|
||||
staleApps.push(stale as string);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2490,7 +2716,7 @@ export async function push(
|
||||
true,
|
||||
);
|
||||
if (stale) {
|
||||
staleApps.push(stale);
|
||||
staleApps.push(stale as string);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2733,6 +2959,21 @@ export async function push(
|
||||
await writeFile(stateTarget, change.after, "utf-8");
|
||||
}
|
||||
continue;
|
||||
} else if (isScriptModulePath(change.path)) {
|
||||
// Module file changed — push the parent script
|
||||
await pushParentScriptForModule(
|
||||
change.path,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
opts.message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
if (stateTarget) {
|
||||
await writeFile(stateTarget, change.after, "utf-8");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stateTarget) {
|
||||
await mkdir(path.dirname(stateTarget), { recursive: true });
|
||||
@@ -2859,6 +3100,17 @@ export async function push(
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
} else if (isScriptModulePath(change.path)) {
|
||||
await pushParentScriptForModule(
|
||||
change.path,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
opts.message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (stateTarget) {
|
||||
await mkdir(path.dirname(stateTarget), { recursive: true });
|
||||
@@ -2901,6 +3153,19 @@ export async function push(
|
||||
if (change.path.endsWith(".lock")) {
|
||||
continue;
|
||||
}
|
||||
if (isScriptModulePath(change.path)) {
|
||||
// Module file deleted — push the parent script (which will now have fewer modules)
|
||||
await pushParentScriptForModule(
|
||||
change.path,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
opts.message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const typ = getTypeStrFromPath(change.path);
|
||||
|
||||
if (typ == "script") {
|
||||
@@ -3276,8 +3541,8 @@ const command = new Command()
|
||||
"Use promotionOverrides from the specified branch instead of regular overrides",
|
||||
)
|
||||
.option(
|
||||
"--branch <branch:string>",
|
||||
"Override the current git branch (works even outside a git repository)",
|
||||
"--branch, --env <branch:string>",
|
||||
"Override the current git branch/environment (works even outside a git repository)",
|
||||
)
|
||||
.action(pull as any)
|
||||
.command("push")
|
||||
@@ -3332,8 +3597,8 @@ const command = new Command()
|
||||
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
|
||||
)
|
||||
.option(
|
||||
"--branch <branch:string>",
|
||||
"Override the current git branch (works even outside a git repository)",
|
||||
"--branch, --env <branch:string>",
|
||||
"Override the current git branch/environment (works even outside a git repository)",
|
||||
)
|
||||
.option("--lint", "Run lint validation before pushing")
|
||||
.option(
|
||||
|
||||
@@ -268,6 +268,7 @@ export async function pushNativeTrigger(
|
||||
is_flow: result.is_flow,
|
||||
service_config: result.service_config,
|
||||
error: result.error,
|
||||
summary: result.summary,
|
||||
};
|
||||
log.debug(`Native trigger ${serviceName}/${externalId} exists on remote`);
|
||||
} catch {
|
||||
@@ -280,6 +281,7 @@ export async function pushNativeTrigger(
|
||||
script_path: localTrigger.script_path,
|
||||
is_flow: localTrigger.is_flow,
|
||||
service_config: localTrigger.service_config,
|
||||
summary: localTrigger.summary,
|
||||
};
|
||||
|
||||
if (remoteTrigger) {
|
||||
@@ -288,11 +290,13 @@ export async function pushNativeTrigger(
|
||||
script_path: localTrigger.script_path,
|
||||
is_flow: localTrigger.is_flow,
|
||||
service_config: localTrigger.service_config,
|
||||
summary: localTrigger.summary,
|
||||
};
|
||||
const remoteCompare = {
|
||||
script_path: remoteTrigger.script_path,
|
||||
is_flow: remoteTrigger.is_flow,
|
||||
service_config: remoteTrigger.service_config,
|
||||
summary: remoteTrigger.summary,
|
||||
};
|
||||
|
||||
if (isSuperset(localCompare, remoteCompare)) {
|
||||
|
||||
@@ -568,11 +568,11 @@ const command = new Command()
|
||||
.action(listRemote as any)
|
||||
.command("bind")
|
||||
.description("Bind the current Git branch to the active workspace")
|
||||
.option("--branch <branch:string>", "Specify branch (defaults to current)")
|
||||
.option("--branch, --env <branch:string>", "Specify branch/environment (defaults to current)")
|
||||
.action((opts) => bind(opts as any, true))
|
||||
.command("unbind")
|
||||
.description("Remove workspace binding from the current Git branch")
|
||||
.option("--branch <branch:string>", "Specify branch (defaults to current)")
|
||||
.option("--branch, --env <branch:string>", "Specify branch/environment (defaults to current)")
|
||||
.action((opts) => bind(opts as any, false))
|
||||
.command("fork")
|
||||
.description("Create a forked workspace")
|
||||
|
||||
@@ -85,6 +85,8 @@ export interface SyncOptions {
|
||||
};
|
||||
};
|
||||
};
|
||||
// Alias for gitBranches - for users who prefer environment-based terminology
|
||||
environments?: SyncOptions["gitBranches"];
|
||||
// Legacy field - deprecated, use gitBranches instead
|
||||
git_branches?: {
|
||||
commonSpecificItems?: {
|
||||
@@ -243,6 +245,18 @@ export async function readConfigFile(): Promise<SyncOptions> {
|
||||
}
|
||||
}
|
||||
|
||||
// Handle environments -> gitBranches alias (permanent alias, not a deprecation)
|
||||
if (conf && "environments" in conf) {
|
||||
if (!conf.gitBranches) {
|
||||
conf.gitBranches = conf.environments as any;
|
||||
} else {
|
||||
log.warn(
|
||||
"⚠️ Both 'environments' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'environments'."
|
||||
);
|
||||
}
|
||||
delete (conf as any).environments;
|
||||
}
|
||||
|
||||
// Handle git_branches to gitBranches migration
|
||||
if (conf && "git_branches" in conf) {
|
||||
if (!conf.gitBranches) {
|
||||
|
||||
@@ -53,6 +53,7 @@ export interface SimplifiedSettings {
|
||||
mute_critical_alerts?: boolean;
|
||||
color?: string;
|
||||
operator_settings?: any;
|
||||
datatable?: any;
|
||||
slack_team_id?: string;
|
||||
slack_name?: string;
|
||||
slack_command_script?: string;
|
||||
@@ -100,6 +101,7 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings {
|
||||
if (settings.mute_critical_alerts !== undefined) result.mute_critical_alerts = settings.mute_critical_alerts;
|
||||
if (settings.color !== undefined) result.color = settings.color;
|
||||
if (settings.operator_settings !== undefined) result.operator_settings = settings.operator_settings;
|
||||
if (settings.datatable !== undefined) result.datatable = settings.datatable;
|
||||
if (settings.slack_team_id !== undefined) result.slack_team_id = settings.slack_team_id;
|
||||
if (settings.slack_name !== undefined) result.slack_name = settings.slack_name;
|
||||
if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script;
|
||||
@@ -192,6 +194,7 @@ export async function pushWorkspaceSettings(
|
||||
mute_critical_alerts: remoteSettings.mute_critical_alerts,
|
||||
color: remoteSettings.color,
|
||||
operator_settings: remoteSettings.operator_settings,
|
||||
datatable: remoteSettings.datatable,
|
||||
slack_team_id: remoteSettings.slack_team_id,
|
||||
slack_name: remoteSettings.slack_name,
|
||||
slack_command_script: remoteSettings.slack_command_script,
|
||||
@@ -382,6 +385,14 @@ export async function pushWorkspaceSettings(
|
||||
});
|
||||
}
|
||||
|
||||
if (!deepEqual(localSettings.datatable, settings.datatable)) {
|
||||
log.debug(`Updating datatable config...`);
|
||||
await wmill.editDataTableConfig({
|
||||
workspace,
|
||||
requestBody: { settings: localSettings.datatable ?? { datatables: {} } },
|
||||
});
|
||||
}
|
||||
|
||||
if (localSettings.slack_command_script != settings.slack_command_script) {
|
||||
log.debug(`Updating slack command script...`);
|
||||
await wmill.editSlackCommand({
|
||||
|
||||
+1956
-1794
File diff suppressed because one or more lines are too long
+3
-1
@@ -39,6 +39,7 @@ import queues from "./commands/queues/queues.ts";
|
||||
import dependencies from "./commands/dependencies/dependencies.ts";
|
||||
import init from "./commands/init/init.ts";
|
||||
import jobs from "./commands/jobs/jobs.ts";
|
||||
import generateMetadata from "./commands/generate-metadata/generate-metadata.ts";
|
||||
import docs from "./commands/docs/docs.ts";
|
||||
import { fetchVersion } from "./core/context.ts";
|
||||
|
||||
@@ -67,7 +68,7 @@ export {
|
||||
workspaceAdd,
|
||||
};
|
||||
|
||||
export const VERSION = "1.653.0";
|
||||
export const VERSION = "1.664.0";
|
||||
|
||||
// Re-exported from constants.ts to maintain backwards compatibility
|
||||
export { WM_FORK_PREFIX } from "./core/constants.ts";
|
||||
@@ -129,6 +130,7 @@ const command = new Command()
|
||||
.command("queues", queues)
|
||||
.command("dependencies", dependencies)
|
||||
.command("jobs", jobs)
|
||||
.command("generate-metadata", generateMetadata)
|
||||
.command("docs", docs)
|
||||
.command("version --version", "Show version information")
|
||||
.action(async (opts: any) => {
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
isRawAppPath,
|
||||
extractResourceName,
|
||||
buildFolderPath,
|
||||
isScriptModulePath,
|
||||
} from "./utils/resource_folders.ts";
|
||||
import type { PermissionedAsContext } from "./core/permissioned_as.ts";
|
||||
|
||||
@@ -261,6 +262,9 @@ export function getTypeStrFromPath(
|
||||
| "settings"
|
||||
| "encryption_key"
|
||||
| "workspace_dependencies" {
|
||||
if (isScriptModulePath(p)) {
|
||||
return "script";
|
||||
}
|
||||
if (isFlowPath(p)) {
|
||||
return "flow";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
/**
|
||||
* Double-linked dependency tree for tracking script imports and propagating staleness.
|
||||
*/
|
||||
|
||||
import { Workspace } from "../commands/workspace/workspace.ts";
|
||||
import * as wmill from "../../gen/services.gen.ts";
|
||||
import type { ScriptLang } from "../../gen/types.gen.ts";
|
||||
import { ScriptLanguage } from "./script_common.ts";
|
||||
import {
|
||||
filterWorkspaceDependencies,
|
||||
generateScriptHash,
|
||||
checkifMetadataUptodate,
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
updateMetadataGlobalLock,
|
||||
} from "./metadata.ts";
|
||||
import { generateHash } from "./utils.ts";
|
||||
|
||||
/**
|
||||
* Diff local scripts against deployed versions, upload only those that differ.
|
||||
* Only uploaded (mismatched) scripts get contentHash set, so flatten() returns
|
||||
* temp_script_refs only for scripts the backend can't resolve from deployed versions.
|
||||
*/
|
||||
export async function uploadScripts(
|
||||
tree: DoubleLinkedDependencyTree,
|
||||
workspace: Workspace
|
||||
): Promise<void> {
|
||||
// Split into scripts vs workspace deps and compute SHA256(content) for each
|
||||
const scriptHashes: Record<string, string> = {};
|
||||
const workspaceDeps: { path: string; language: ScriptLang; name?: string; hash: string }[] = [];
|
||||
|
||||
for (const path of tree.allPaths()) {
|
||||
const content = tree.getContent(path);
|
||||
const itemType = tree.getItemType(path);
|
||||
|
||||
if (itemType === "dependencies") {
|
||||
// Empty string is valid for workspace deps (means "no deps") — only skip undefined
|
||||
if (content === undefined) continue;
|
||||
const info = workspaceDependenciesPathToLanguageAndFilename(path);
|
||||
if (info) {
|
||||
const hash = await generateHash(content);
|
||||
workspaceDeps.push({ path, language: info.language as ScriptLang, name: info.name, hash });
|
||||
}
|
||||
} else if (itemType === "script") {
|
||||
if (!content) continue;
|
||||
const hash = await generateHash(content);
|
||||
scriptHashes[path] = hash;
|
||||
}
|
||||
// Skip inline_script, flow, app — they don't need temp storage uploads
|
||||
}
|
||||
|
||||
if (Object.keys(scriptHashes).length === 0 && workspaceDeps.length === 0) return;
|
||||
|
||||
// Single batch query: find which scripts/deps differ from deployed versions
|
||||
const mismatched = await wmill.diffRawScriptsWithDeployed({
|
||||
workspace: workspace.workspaceId,
|
||||
requestBody: {
|
||||
scripts: scriptHashes,
|
||||
workspace_deps: workspaceDeps,
|
||||
},
|
||||
});
|
||||
|
||||
// Upload only mismatched scripts to temp storage
|
||||
for (const path of mismatched) {
|
||||
const content = tree.getContent(path);
|
||||
const itemType = tree.getItemType(path);
|
||||
|
||||
if (itemType === "dependencies") {
|
||||
// Workspace deps don't need temp storage — just mark as mismatched.
|
||||
// Empty string is valid (means the dep file was emptied locally).
|
||||
if (content !== undefined) {
|
||||
tree.setContentHash(path, "mismatched");
|
||||
}
|
||||
} else if (content) {
|
||||
const hash = await wmill.storeRawScriptTemp({
|
||||
workspace: workspace.workspaceId,
|
||||
requestBody: content,
|
||||
});
|
||||
tree.setContentHash(path, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type ItemType = "script" | "inline_script" | "flow" | "app" | "dependencies";
|
||||
|
||||
interface DependencyNode {
|
||||
content: string;
|
||||
stalenessHash: string; // Hash for staleness detection (includes deps, content, metadata)
|
||||
contentHash?: string; // Hash for temp storage lookup (content only)
|
||||
language: ScriptLanguage;
|
||||
metadata: string;
|
||||
imports: Set<string>;
|
||||
importedBy: Set<string>;
|
||||
staleReason?: string;
|
||||
// Item metadata for generate-metadata command
|
||||
itemType: ItemType;
|
||||
folder: string; // Folder path (for flows/apps) or remote path (for scripts)
|
||||
originalPath: string; // Original path passed to handler (with extension for scripts)
|
||||
isRawApp?: boolean; // Only set for apps
|
||||
isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale)
|
||||
}
|
||||
|
||||
export class DoubleLinkedDependencyTree {
|
||||
private nodes: Map<string, DependencyNode> = new Map();
|
||||
private workspaceDeps: Record<string, string> = {};
|
||||
|
||||
setWorkspaceDeps(deps: Record<string, string>): void {
|
||||
this.workspaceDeps = deps;
|
||||
}
|
||||
|
||||
async addNode(
|
||||
path: string,
|
||||
content: string,
|
||||
language: ScriptLanguage,
|
||||
metadata: string,
|
||||
imports: string[],
|
||||
itemType: ItemType,
|
||||
folder: string,
|
||||
originalPath: string,
|
||||
isDirectlyStale: boolean,
|
||||
isRawApp?: boolean
|
||||
): Promise<void> {
|
||||
const hasWorkspaceDeps = itemType === "script" || itemType === "inline_script";
|
||||
const filteredDeps = hasWorkspaceDeps
|
||||
? filterWorkspaceDependencies(this.workspaceDeps, content, language)
|
||||
: {};
|
||||
const stalenessHash = await generateScriptHash({}, content, metadata);
|
||||
|
||||
if (!this.nodes.has(path)) {
|
||||
this.nodes.set(path, {
|
||||
content: "", stalenessHash: "", language: "deno", metadata: "",
|
||||
imports: new Set(), importedBy: new Set(),
|
||||
itemType: "script", folder: "", originalPath: "", isDirectlyStale: false,
|
||||
});
|
||||
}
|
||||
const node = this.nodes.get(path)!;
|
||||
node.content = content;
|
||||
node.stalenessHash = stalenessHash;
|
||||
node.language = language;
|
||||
node.metadata = metadata;
|
||||
node.itemType = itemType;
|
||||
node.folder = folder;
|
||||
node.originalPath = originalPath;
|
||||
node.isDirectlyStale = isDirectlyStale;
|
||||
node.isRawApp = isRawApp;
|
||||
|
||||
// Create nodes for referenced workspace deps with content and language.
|
||||
const filteredDepsPaths = Object.keys(filteredDeps);
|
||||
for (const depsPath of filteredDepsPaths) {
|
||||
if (!this.nodes.has(depsPath)) {
|
||||
const depsInfo = workspaceDependenciesPathToLanguageAndFilename(depsPath);
|
||||
const contentHash = await generateHash(filteredDeps[depsPath] + depsPath);
|
||||
const isUpToDate = await checkifMetadataUptodate(depsPath, contentHash, undefined);
|
||||
this.nodes.set(depsPath, {
|
||||
content: filteredDeps[depsPath],
|
||||
stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "",
|
||||
imports: new Set(), importedBy: new Set(),
|
||||
itemType: "dependencies", folder: "", originalPath: depsPath,
|
||||
isDirectlyStale: !isUpToDate,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const allImports = [...imports, ...filteredDepsPaths];
|
||||
for (const importPath of allImports) {
|
||||
node.imports.add(importPath);
|
||||
|
||||
if (!this.nodes.has(importPath)) {
|
||||
this.nodes.set(importPath, {
|
||||
content: "", stalenessHash: "", language: "deno", metadata: "",
|
||||
imports: new Set(), importedBy: new Set(),
|
||||
itemType: "script", folder: "", originalPath: "", isDirectlyStale: false,
|
||||
});
|
||||
}
|
||||
this.nodes.get(importPath)!.importedBy.add(path);
|
||||
}
|
||||
}
|
||||
|
||||
getContent(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.content;
|
||||
}
|
||||
|
||||
getStalenessHash(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.stalenessHash;
|
||||
}
|
||||
|
||||
getContentHash(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.contentHash;
|
||||
}
|
||||
|
||||
setContentHash(path: string, hash: string): void {
|
||||
const node = this.nodes.get(path);
|
||||
if (node) {
|
||||
node.contentHash = hash;
|
||||
}
|
||||
}
|
||||
|
||||
getLanguage(path: string): ScriptLanguage | undefined {
|
||||
return this.nodes.get(path)?.language;
|
||||
}
|
||||
|
||||
getMetadata(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.metadata;
|
||||
}
|
||||
|
||||
getStaleReason(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.staleReason;
|
||||
}
|
||||
|
||||
getItemType(path: string): ItemType | undefined {
|
||||
return this.nodes.get(path)?.itemType;
|
||||
}
|
||||
|
||||
getFolder(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.folder;
|
||||
}
|
||||
|
||||
getIsRawApp(path: string): boolean | undefined {
|
||||
return this.nodes.get(path)?.isRawApp;
|
||||
}
|
||||
|
||||
getIsDirectlyStale(path: string): boolean {
|
||||
return this.nodes.get(path)?.isDirectlyStale ?? false;
|
||||
}
|
||||
|
||||
getOriginalPath(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.originalPath;
|
||||
}
|
||||
|
||||
getImports(path: string): Set<string> | undefined {
|
||||
return this.nodes.get(path)?.imports;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this node has been marked stale (directly or transitively).
|
||||
*/
|
||||
isStale(path: string): boolean {
|
||||
return this.nodes.get(path)?.staleReason !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutates the tree by removing all nodes that are not stale.
|
||||
* Uses BFS on reverse graph (importedBy) to find all stale scripts.
|
||||
* Starts from nodes with isDirectlyStale=true.
|
||||
*/
|
||||
propagateStaleness(): void {
|
||||
// Collect directly stale nodes
|
||||
const directlyStale = new Set<string>();
|
||||
for (const [path, node] of this.nodes.entries()) {
|
||||
if (node.isDirectlyStale) {
|
||||
directlyStale.add(path);
|
||||
node.staleReason = "content changed";
|
||||
}
|
||||
}
|
||||
|
||||
const allStale = new Set(directlyStale);
|
||||
const queue = [...directlyStale];
|
||||
const visited = new Set<string>();
|
||||
|
||||
while (queue.length > 0) {
|
||||
const scriptPath = queue.shift()!;
|
||||
if (visited.has(scriptPath)) continue;
|
||||
visited.add(scriptPath);
|
||||
|
||||
const node = this.nodes.get(scriptPath);
|
||||
if (!node) continue;
|
||||
|
||||
for (const importer of node.importedBy) {
|
||||
if (!allStale.has(importer)) {
|
||||
allStale.add(importer);
|
||||
queue.push(importer);
|
||||
// Set reason for transitively stale scripts
|
||||
const importerNode = this.nodes.get(importer);
|
||||
if (importerNode) importerNode.staleReason = `depends on ${scriptPath}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks all transitive imports for a node, calling the callback for each.
|
||||
* Callback may return true to stop traversing that branch.
|
||||
*/
|
||||
traverseTransitive(scriptPath: string, callback: (importPath: string, node: DependencyNode) => boolean | void): void {
|
||||
const queue = [scriptPath];
|
||||
const visited = new Set<string>();
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift()!;
|
||||
if (visited.has(current)) continue;
|
||||
visited.add(current);
|
||||
|
||||
const node = this.nodes.get(current);
|
||||
if (!node) continue;
|
||||
|
||||
for (const importPath of node.imports) {
|
||||
const importNode = this.nodes.get(importPath);
|
||||
if (importNode) {
|
||||
const stop = callback(importPath, importNode);
|
||||
if (!stop) {
|
||||
queue.push(importPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
allPaths(): IterableIterator<string> {
|
||||
return this.nodes.keys();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns paths of all stale nodes (those with a staleReason).
|
||||
*/
|
||||
*stalePaths(): IterableIterator<string> {
|
||||
for (const [path, node] of this.nodes.entries()) {
|
||||
if (node.staleReason) {
|
||||
yield path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
has(path: string): boolean {
|
||||
return this.nodes.has(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns workspace deps that were uploaded as mismatched with remote.
|
||||
* These need to be passed as raw_workspace_dependencies in job args
|
||||
* so the backend uses local content instead of deployed.
|
||||
*/
|
||||
getMismatchedWorkspaceDeps(): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
for (const [path, node] of this.nodes.entries()) {
|
||||
if (node.itemType === "dependencies" && node.contentHash && node.content !== undefined) {
|
||||
result[path] = node.content;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns path → contentHash for all transitive imports that have been uploaded.
|
||||
* Must be called after uploadScripts() has populated contentHash values.
|
||||
*/
|
||||
getTempScriptRefs(scriptPath: string): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
this.traverseTransitive(scriptPath, (_path, node) => {
|
||||
if (node.contentHash) {
|
||||
result[_path] = node.contentHash;
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist workspace dep hashes to wmill-lock.yaml so getRawWorkspaceDependencies
|
||||
* considers them up-to-date on the next run.
|
||||
*/
|
||||
async persistDepsHashes(depsPaths: string[]): Promise<void> {
|
||||
for (const path of depsPaths) {
|
||||
const node = this.nodes.get(path);
|
||||
if (node?.itemType === "dependencies" && node.content !== undefined) {
|
||||
const hash = await generateHash(node.content + path);
|
||||
await updateMetadataGlobalLock(path, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.nodes.size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFile, stat } from "node:fs/promises";
|
||||
import type { SyncCodebase } from "./codebase.ts";
|
||||
import { parseMetadataFileIfExists } from "./metadata.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
import { findCodebase } from "../commands/sync/sync.ts";
|
||||
import type { LocalScriptInfo } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import type { RawScript } from "../../../gen/types.gen.ts";
|
||||
|
||||
export class UnsupportedLocalPathScriptPreviewError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "UnsupportedLocalPathScriptPreviewError";
|
||||
}
|
||||
}
|
||||
|
||||
async function readOptionalLock(scriptPath: string): Promise<string | undefined> {
|
||||
try {
|
||||
return await readFile(scriptPath + ".script.lock", "utf-8");
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptionalLock(lock: string | undefined): string | undefined {
|
||||
return typeof lock === "string" && lock.trim() === "" ? undefined : lock;
|
||||
}
|
||||
|
||||
async function bundleSingleFileCodebaseScript(
|
||||
filePath: string,
|
||||
codebase: SyncCodebase
|
||||
): Promise<string> {
|
||||
if (codebase.customBundler) {
|
||||
// Pass the script path as a positional shell argument so existing shell-based
|
||||
// custom bundlers still work without interpolating the path into the command.
|
||||
return execFileSync(
|
||||
"sh",
|
||||
["-lc", `${codebase.customBundler} "$1"`, "sh", filePath],
|
||||
{
|
||||
maxBuffer: 1024 * 1024 * 50,
|
||||
}
|
||||
).toString();
|
||||
}
|
||||
|
||||
const esbuild = await import("esbuild");
|
||||
const out = await esbuild.build({
|
||||
entryPoints: [filePath],
|
||||
// Inline rawscripts are executed through the standard module wrapper,
|
||||
// so the bundle must expose `main` as an ESM export.
|
||||
format: "esm",
|
||||
bundle: true,
|
||||
write: false,
|
||||
external: codebase.external,
|
||||
inject: codebase.inject,
|
||||
define: codebase.define,
|
||||
loader: codebase.loader ?? { ".node": "file" },
|
||||
outdir: "/",
|
||||
platform: "node",
|
||||
packages: "bundle",
|
||||
target: "esnext",
|
||||
banner: codebase.banner,
|
||||
});
|
||||
|
||||
if (out.outputFiles.length === 0) {
|
||||
throw new Error(`No output files found for ${filePath}`);
|
||||
}
|
||||
if (out.outputFiles.length > 1) {
|
||||
throw new UnsupportedLocalPathScriptPreviewError(
|
||||
`Local PathScript ${filePath} requires a multi-file bundle, which flow preview/dev cannot inline yet`
|
||||
);
|
||||
}
|
||||
if (Array.isArray(codebase.assets) && codebase.assets.length > 0) {
|
||||
throw new UnsupportedLocalPathScriptPreviewError(
|
||||
`Local PathScript ${filePath} requires codebase assets, which flow preview/dev cannot inline yet`
|
||||
);
|
||||
}
|
||||
|
||||
return out.outputFiles[0].text;
|
||||
}
|
||||
|
||||
export function createPreviewLocalScriptReader(opts: {
|
||||
exts: string[];
|
||||
defaultTs?: "bun" | "deno";
|
||||
codebases: SyncCodebase[];
|
||||
}): (scriptPath: string) => Promise<LocalScriptInfo | undefined> {
|
||||
return async (scriptPath) => {
|
||||
const localScript = await resolvePreviewLocalScriptState(scriptPath, opts);
|
||||
if (!localScript) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const content = localScript.codebase
|
||||
? await bundleSingleFileCodebaseScript(localScript.filePath, localScript.codebase)
|
||||
: localScript.content;
|
||||
|
||||
return {
|
||||
content,
|
||||
language: localScript.language,
|
||||
lock: localScript.lock,
|
||||
tag: localScript.tag,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export type PreviewLocalScriptState = {
|
||||
filePath: string;
|
||||
content: string;
|
||||
language: RawScript["language"];
|
||||
lock?: string;
|
||||
tag?: string;
|
||||
codebase?: SyncCodebase;
|
||||
codebaseDigest?: string;
|
||||
};
|
||||
|
||||
export async function resolvePreviewLocalScriptState(
|
||||
scriptPath: string,
|
||||
opts: {
|
||||
exts: string[];
|
||||
defaultTs?: "bun" | "deno";
|
||||
codebases: SyncCodebase[];
|
||||
}
|
||||
): Promise<PreviewLocalScriptState | undefined> {
|
||||
for (const ext of opts.exts) {
|
||||
const filePath = scriptPath + ext;
|
||||
let fileStat;
|
||||
try {
|
||||
fileStat = await stat(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!fileStat.isFile()) continue;
|
||||
|
||||
const language = inferContentTypeFromFilePath(filePath, opts.defaultTs);
|
||||
const metadata = await parseMetadataFileIfExists(scriptPath);
|
||||
const rawLock = metadata?.payload?.lock ?? (await readOptionalLock(scriptPath));
|
||||
const codebase =
|
||||
language === "bun" ? findCodebase(filePath, opts.codebases) : undefined;
|
||||
|
||||
return {
|
||||
filePath,
|
||||
content: await readFile(filePath, "utf-8"),
|
||||
language,
|
||||
lock: normalizeOptionalLock(rawLock),
|
||||
tag: metadata?.payload?.tag,
|
||||
codebase,
|
||||
codebaseDigest: codebase
|
||||
? await codebase.getDigest(
|
||||
Array.isArray(codebase.assets) && codebase.assets.length > 0
|
||||
)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
+425
-107
@@ -5,7 +5,8 @@ import * as log from "../core/log.ts";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { yamlParseFile } from "./yaml.ts";
|
||||
import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import {
|
||||
ScriptMetadata,
|
||||
@@ -15,19 +16,23 @@ import { Workspace } from "../commands/workspace/workspace.ts";
|
||||
import {
|
||||
ScriptLanguage,
|
||||
workspaceDependenciesLanguages,
|
||||
languageNeedsLock,
|
||||
} from "./script_common.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts";
|
||||
import { findCodebase, yamlOptions } from "../commands/sync/sync.ts";
|
||||
import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts";
|
||||
|
||||
import { SyncCodebase } from "./codebase.ts";
|
||||
import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts";
|
||||
import { getIsWin } from "./utils.ts";
|
||||
import { extractRelativeImports } from "./relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "./dependency_tree.ts";
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
const _parserCache = new Map<string, Promise<any>>();
|
||||
|
||||
function loadParser(pkgName: string): Promise<any> {
|
||||
export function loadParser(pkgName: string): Promise<any> {
|
||||
let p = _parserCache.get(pkgName);
|
||||
if (!p) {
|
||||
p = (async () => {
|
||||
@@ -35,7 +40,7 @@ function loadParser(pkgName: string): Promise<any> {
|
||||
const wasmPath = _require.resolve(
|
||||
`${pkgName}/windmill_parser_wasm_bg.wasm`
|
||||
);
|
||||
await mod.default(readFileSync(wasmPath));
|
||||
await mod.default({ module_or_path: readFileSync(wasmPath) });
|
||||
return mod;
|
||||
})();
|
||||
_parserCache.set(pkgName, p);
|
||||
@@ -50,27 +55,28 @@ export class LockfileGenerationError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateAllMetadata() {}
|
||||
|
||||
export async function getRawWorkspaceDependencies(): Promise<Record<string, string>> {
|
||||
export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise<Record<string, string>> {
|
||||
const rawWorkspaceDeps: Record<string, string> = {};
|
||||
|
||||
|
||||
try {
|
||||
const entries = await readdir("dependencies", { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) continue;
|
||||
|
||||
|
||||
const filePath = `dependencies/${entry.name}`;
|
||||
const content = await readFile(filePath, "utf-8");
|
||||
|
||||
|
||||
// Find matching language
|
||||
for (const lang of workspaceDependenciesLanguages) {
|
||||
if (entry.name.endsWith(lang.filename)) {
|
||||
// Check if out of sync
|
||||
const contentHash = await generateHash(content + filePath);
|
||||
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
|
||||
|
||||
if (!isUpToDate) {
|
||||
if (legacyBehaviour) {
|
||||
const contentHash = await generateHash(content + filePath);
|
||||
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
|
||||
if (!isUpToDate) {
|
||||
rawWorkspaceDeps[filePath] = content;
|
||||
}
|
||||
} else {
|
||||
rawWorkspaceDeps[filePath] = content;
|
||||
}
|
||||
break;
|
||||
@@ -184,15 +190,22 @@ export async function generateScriptMetadataInternal(
|
||||
noStaleMessage: boolean,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
codebases: SyncCodebase[],
|
||||
justUpdateMetadataLock?: boolean
|
||||
justUpdateMetadataLock?: boolean,
|
||||
legacyBehaviour?: boolean,
|
||||
tree?: DoubleLinkedDependencyTree
|
||||
): Promise<string | undefined> {
|
||||
const remotePath = scriptPath
|
||||
.substring(0, scriptPath.indexOf("."))
|
||||
.replaceAll(SEP, "/");
|
||||
// Detect folder layout: my_script__mod/script.ts
|
||||
const isFolderLayout = isModuleEntryPoint(scriptPath);
|
||||
|
||||
// remotePath is the Windmill API path (e.g., "u/admin/my_script")
|
||||
const remotePath = isFolderLayout
|
||||
? getScriptBasePathFromModulePath(scriptPath)!.replaceAll(SEP, "/")
|
||||
: scriptPath.substring(0, scriptPath.indexOf(".")).replaceAll(SEP, "/");
|
||||
|
||||
const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
|
||||
|
||||
|
||||
// For folder layout, parseMetadataFile is called with remotePath which
|
||||
// will find __mod/script.yaml via the folder layout fallback
|
||||
const metadataWithType = await parseMetadataFile(
|
||||
remotePath,
|
||||
undefined,
|
||||
@@ -208,22 +221,76 @@ export async function generateScriptMetadataInternal(
|
||||
language
|
||||
);
|
||||
|
||||
// Compute the module folder path early so we can include module hashes in stale check
|
||||
const moduleFolderPath = isFolderLayout
|
||||
? path.dirname(scriptPath)
|
||||
: scriptPath.substring(0, scriptPath.indexOf(".")) + getModuleFolderSuffix();
|
||||
|
||||
// Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically
|
||||
let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent);
|
||||
const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory();
|
||||
|
||||
if (await checkifMetadataUptodate(remotePath, hash, undefined)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return `${remotePath} (${language})`;
|
||||
// In non-legacy mode, workspace deps are tracked via the tree — exclude from hash
|
||||
const depsForHash = (!legacyBehaviour && tree) ? {} : filteredRawWorkspaceDependencies;
|
||||
let hash = await generateScriptHash(depsForHash, scriptContent, metadataContent);
|
||||
|
||||
// Compute per-module hashes for stale detection (like flow inline scripts)
|
||||
let moduleHashes: Record<string, string> = {};
|
||||
if (hasModules) {
|
||||
moduleHashes = await computeModuleHashes(
|
||||
moduleFolderPath, opts.defaultTs, (!legacyBehaviour && tree) ? {} : rawWorkspaceDependencies, isFolderLayout
|
||||
);
|
||||
}
|
||||
const hasModuleHashes = Object.keys(moduleHashes).length > 0;
|
||||
|
||||
// If modules exist, combine main script hash + module hashes into a meta-hash
|
||||
let checkHash = hash;
|
||||
let checkSubpath: string | undefined;
|
||||
if (hasModuleHashes) {
|
||||
const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b));
|
||||
checkHash = await generateHash(hash + JSON.stringify(sortedEntries));
|
||||
checkSubpath = SCRIPT_TOP_HASH;
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
const conf = await readLockfile();
|
||||
|
||||
// Use checkHash (includes module hashes) so module changes are detected as stale
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath));
|
||||
|
||||
// New behaviour: tree-based dependency tracking
|
||||
if (!legacyBehaviour && tree) {
|
||||
if (dryRun) {
|
||||
// First pass: populate tree with script and its imports
|
||||
const imports = await extractRelativeImports(scriptContent, remotePath, language);
|
||||
await tree.addNode(remotePath, scriptContent, language, metadataContent, imports, "script", remotePath, scriptPath, isDirectlyStale);
|
||||
return;
|
||||
}
|
||||
// Second pass: proceed to generate (caller verified this script is stale via tree)
|
||||
} else {
|
||||
// Legacy behaviour: use existing staleness check
|
||||
if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
let detail = `${remotePath} (${language})`;
|
||||
if (hasModuleHashes) {
|
||||
const changed: string[] = [];
|
||||
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
|
||||
if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) {
|
||||
changed.push(modulePath);
|
||||
}
|
||||
}
|
||||
if (changed.length > 0) {
|
||||
detail += ` [changed modules: ${changed.join(", ")}]`;
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock && !noStaleMessage) {
|
||||
log.info(colors.gray(`Generating metadata for ${scriptPath}`));
|
||||
}
|
||||
|
||||
@@ -245,36 +312,97 @@ export async function generateScriptMetadataInternal(
|
||||
const hasCodebase = findCodebase(scriptPath, codebases) != undefined;
|
||||
|
||||
if (!hasCodebase) {
|
||||
const tempScriptRefs = tree?.getTempScriptRefs(remotePath);
|
||||
const lockPathOverride = isFolderLayout
|
||||
? path.dirname(scriptPath) + "/script.lock"
|
||||
: undefined;
|
||||
await updateScriptLock(
|
||||
workspace,
|
||||
scriptContent,
|
||||
language,
|
||||
remotePath,
|
||||
metadataParsedContent,
|
||||
filteredRawWorkspaceDependencies
|
||||
filteredRawWorkspaceDependencies,
|
||||
tempScriptRefs,
|
||||
lockPathOverride,
|
||||
);
|
||||
} else {
|
||||
metadataParsedContent.lock = "";
|
||||
}
|
||||
|
||||
// Generate locks for modules in __mod/ folder
|
||||
if (hasModules) {
|
||||
// Identify which modules changed by comparing per-module hashes
|
||||
let changedModules: string[] | undefined;
|
||||
if (hasModuleHashes) {
|
||||
changedModules = [];
|
||||
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
|
||||
if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) {
|
||||
changedModules.push(modulePath);
|
||||
}
|
||||
}
|
||||
if (changedModules.length === 0) {
|
||||
changedModules = undefined; // no modules changed, skip lock regeneration
|
||||
}
|
||||
}
|
||||
await updateModuleLocks(
|
||||
workspace, moduleFolderPath, "", remotePath,
|
||||
rawWorkspaceDependencies, opts.defaultTs, changedModules,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
|
||||
if (isFolderLayout) {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + getModuleFolderSuffix() + "/script.lock";
|
||||
} else {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
|
||||
}
|
||||
}
|
||||
let metaPath = remotePath + ".script.yaml";
|
||||
let newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
|
||||
if (metadataWithType.isJson) {
|
||||
metaPath = remotePath + ".script.json";
|
||||
newMetadataContent = JSON.stringify(metadataParsedContent);
|
||||
|
||||
// Write metadata back to the correct path
|
||||
let metaPath: string;
|
||||
let newMetadataContent: string;
|
||||
if (isFolderLayout) {
|
||||
if (metadataWithType.isJson) {
|
||||
metaPath = path.dirname(scriptPath) + "/script.json";
|
||||
newMetadataContent = JSON.stringify(metadataParsedContent);
|
||||
} else {
|
||||
metaPath = path.dirname(scriptPath) + "/script.yaml";
|
||||
newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
|
||||
}
|
||||
} else {
|
||||
if (metadataWithType.isJson) {
|
||||
metaPath = remotePath + ".script.json";
|
||||
newMetadataContent = JSON.stringify(metadataParsedContent);
|
||||
} else {
|
||||
metaPath = remotePath + ".script.yaml";
|
||||
newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataContentUsedForHash = newMetadataContent;
|
||||
|
||||
hash = await generateScriptHash(
|
||||
filteredRawWorkspaceDependencies,
|
||||
depsForHash,
|
||||
scriptContent,
|
||||
metadataContentUsedForHash
|
||||
);
|
||||
await updateMetadataGlobalLock(remotePath, hash);
|
||||
|
||||
// Store hashes in wmill-lock.yaml
|
||||
if (hasModuleHashes) {
|
||||
// Use per-module hash tracking (like flow inline scripts)
|
||||
const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b));
|
||||
const metaHash = await generateHash(hash + JSON.stringify(sortedEntries));
|
||||
await clearGlobalLock(remotePath);
|
||||
await updateMetadataGlobalLock(remotePath, metaHash, SCRIPT_TOP_HASH);
|
||||
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
|
||||
await updateMetadataGlobalLock(remotePath, moduleHash, modulePath);
|
||||
}
|
||||
} else {
|
||||
await updateMetadataGlobalLock(remotePath, hash);
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
await writeFile(metaPath, newMetadataContent, "utf-8");
|
||||
}
|
||||
@@ -300,11 +428,9 @@ export async function updateScriptSchema(
|
||||
} else {
|
||||
delete metadataContent.has_preprocessor;
|
||||
}
|
||||
if (result.no_main_func) {
|
||||
metadataContent.no_main_func = result.no_main_func;
|
||||
} else {
|
||||
delete metadataContent.no_main_func;
|
||||
}
|
||||
// auto_kind is intentionally not written to metadata — it is auto-detected
|
||||
// by the parser at deploy time from script content.
|
||||
delete metadataContent.auto_kind;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -329,6 +455,7 @@ const LANG_ANNOTATION_CONFIG: Partial<
|
||||
nativets: { comment: "//", keyword: "package_json" },
|
||||
go: { comment: "//", keyword: "go_mod" },
|
||||
php: { comment: "//", keyword: "composer_json" },
|
||||
powershell: { comment: "#", keyword: "modules_json" },
|
||||
};
|
||||
|
||||
export function extractWorkspaceDepsAnnotation(
|
||||
@@ -410,6 +537,7 @@ export async function computeLockCacheKey(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string> {
|
||||
const annotation = extractWorkspaceDepsAnnotation(scriptContent, language);
|
||||
const annotationStr = annotation
|
||||
@@ -417,7 +545,10 @@ export async function computeLockCacheKey(
|
||||
: "none";
|
||||
const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort();
|
||||
const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";");
|
||||
return await generateHash(`${language}|${annotationStr}|${depsStr}`);
|
||||
const tempRefsStr = tempScriptRefs
|
||||
? Object.keys(tempScriptRefs).sort().map((k) => `${k}=${tempScriptRefs[k]}`).join(";")
|
||||
: "";
|
||||
return await generateHash(`${language}|${annotationStr}|${depsStr}|${tempRefsStr}`);
|
||||
}
|
||||
|
||||
const lockCache = new Map<string, string>();
|
||||
@@ -432,13 +563,15 @@ async function fetchScriptLock(
|
||||
language: ScriptLanguage,
|
||||
remotePath: string,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string> {
|
||||
const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0;
|
||||
const cacheKey = hasRawDeps
|
||||
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies)
|
||||
const hasTempRefs = tempScriptRefs && Object.keys(tempScriptRefs).length > 0;
|
||||
const cacheKey = (hasRawDeps || hasTempRefs)
|
||||
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies, tempScriptRefs)
|
||||
: undefined;
|
||||
if (cacheKey && lockCache.has(cacheKey)) {
|
||||
log.info(`Using cached lockfile for ${remotePath}`);
|
||||
log.debug(`Using cached lockfile for ${remotePath}`);
|
||||
return lockCache.get(cacheKey)!;
|
||||
}
|
||||
|
||||
@@ -463,6 +596,8 @@ async function fetchScriptLock(
|
||||
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
|
||||
? rawWorkspaceDependencies : null,
|
||||
entrypoint: remotePath,
|
||||
temp_script_refs: tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? tempScriptRefs : null,
|
||||
}),
|
||||
}
|
||||
);
|
||||
@@ -502,11 +637,14 @@ async function updateScriptLock(
|
||||
language: ScriptLanguage,
|
||||
remotePath: string,
|
||||
metadataContent: Record<string, any>,
|
||||
rawWorkspaceDependencies: Record<string, string>
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>,
|
||||
lockPathOverride?: string,
|
||||
): Promise<void> {
|
||||
if (
|
||||
!(
|
||||
workspaceDependenciesLanguages.some((l) => l.language == language) ||
|
||||
(workspaceDependenciesLanguages.some((l) => l.language == language) &&
|
||||
language !== "powershell") ||
|
||||
language == "deno" ||
|
||||
language == "rust" ||
|
||||
language == "ansible"
|
||||
@@ -518,7 +656,7 @@ async function updateScriptLock(
|
||||
|
||||
if (Object.keys(rawWorkspaceDependencies).length > 0) {
|
||||
const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', ');
|
||||
log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
|
||||
log.debug(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
|
||||
}
|
||||
|
||||
const lock = await fetchScriptLock(
|
||||
@@ -527,9 +665,10 @@ async function updateScriptLock(
|
||||
language,
|
||||
remotePath,
|
||||
rawWorkspaceDependencies,
|
||||
tempScriptRefs
|
||||
);
|
||||
|
||||
const lockPath = remotePath + ".script.lock";
|
||||
const lockPath = lockPathOverride ?? remotePath + ".script.lock";
|
||||
if (lock != "") {
|
||||
await writeFile(lockPath, lock, "utf-8");
|
||||
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
|
||||
@@ -545,6 +684,82 @@ async function updateScriptLock(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate locks for all module files in a __mod/ directory.
|
||||
* Recursively walks the directory and generates a lock for each module
|
||||
* whose language requires one.
|
||||
*/
|
||||
async function updateModuleLocks(
|
||||
workspace: Workspace,
|
||||
dirPath: string,
|
||||
relPrefix: string,
|
||||
scriptRemotePath: string,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
changedModules?: string[],
|
||||
): Promise<void> {
|
||||
const entries = readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await updateModuleLocks(workspace, fullPath, relPath, scriptRemotePath, rawWorkspaceDependencies, defaultTs, changedModules);
|
||||
} else if (entry.isFile()
|
||||
&& !entry.name.endsWith(".lock")
|
||||
// In folder layout, skip entry point files (script.{ext}, script.yaml, script.json, script.lock)
|
||||
&& !(relPrefix === "" && entry.name.startsWith("script."))
|
||||
) {
|
||||
let modLanguage: ScriptLanguage;
|
||||
try {
|
||||
modLanguage = inferContentTypeFromFilePath(entry.name, defaultTs);
|
||||
} catch {
|
||||
continue; // skip files with unrecognized extensions
|
||||
}
|
||||
|
||||
if (!languageNeedsLock(modLanguage)) continue;
|
||||
|
||||
// Skip unchanged modules when per-module hash tracking is active
|
||||
if (changedModules) {
|
||||
const normalizedRelPath = normalizeLockPath(relPath);
|
||||
if (!changedModules.includes(normalizedRelPath)) continue;
|
||||
}
|
||||
|
||||
const moduleContent = readFileSync(fullPath, "utf-8");
|
||||
const moduleRemotePath = scriptRemotePath + "/" + relPath;
|
||||
|
||||
log.debug(`Generating lock for module ${relPath}`);
|
||||
|
||||
try {
|
||||
const lock = await fetchScriptLock(
|
||||
workspace,
|
||||
moduleContent,
|
||||
modLanguage,
|
||||
moduleRemotePath,
|
||||
rawWorkspaceDependencies,
|
||||
);
|
||||
|
||||
const baseName = entry.name.replace(/\.[^.]+$/, '');
|
||||
const lockPath = path.join(dirPath, baseName + ".lock");
|
||||
if (lock != "") {
|
||||
writeFileSync(lockPath, lock, "utf-8");
|
||||
} else {
|
||||
try {
|
||||
if (existsSync(lockPath)) {
|
||||
const { rm: rmAsync } = await import("node:fs/promises");
|
||||
await rmAsync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.info(colors.yellow(`Failed to generate lock for module ${relPath}: ${e}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// below functions copied from Windmill's FE inferArgs function. TODO: refactor //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -556,7 +771,7 @@ export async function inferSchema(
|
||||
): Promise<{
|
||||
schema: any;
|
||||
has_preprocessor: boolean | undefined;
|
||||
no_main_func: boolean | undefined;
|
||||
auto_kind: string | undefined;
|
||||
}> {
|
||||
let inferedSchema: any;
|
||||
if (language === "python3") {
|
||||
@@ -666,7 +881,7 @@ export async function inferSchema(
|
||||
return {
|
||||
schema: defaultScriptMetadata().schema,
|
||||
has_preprocessor: false,
|
||||
no_main_func: false,
|
||||
auto_kind: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -691,6 +906,11 @@ export async function inferSchema(
|
||||
|
||||
argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]);
|
||||
|
||||
// For T | T[] detection for debouncing arg accumulation
|
||||
if ((arg as any).otyp && (arg as any).otyp.includes('[') && (arg as any).otyp.includes('|')) {
|
||||
currentSchema.properties[arg.name].originalType = (arg as any).otyp
|
||||
}
|
||||
|
||||
currentSchema.properties[arg.name].default = arg.default;
|
||||
|
||||
if (!arg.has_default && !currentSchema.required.includes(arg.name)) {
|
||||
@@ -701,7 +921,7 @@ export async function inferSchema(
|
||||
return {
|
||||
schema: currentSchema,
|
||||
has_preprocessor: inferedSchema.has_preprocessor,
|
||||
no_main_func: inferedSchema.no_main_func,
|
||||
auto_kind: inferedSchema.auto_kind,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -737,6 +957,38 @@ export function replaceLock(o?: { lock?: string | string[] }) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function parseMetadataFileIfExists(
|
||||
scriptPath: string
|
||||
): Promise<{ isJson: boolean; payload: any; path: string } | undefined> {
|
||||
let metadataFilePath = scriptPath + ".script.json";
|
||||
try {
|
||||
await stat(metadataFilePath);
|
||||
const payload = JSON.parse(await readFile(metadataFilePath, "utf-8"));
|
||||
replaceLock(payload);
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload,
|
||||
isJson: true,
|
||||
};
|
||||
} catch {
|
||||
try {
|
||||
metadataFilePath = scriptPath + ".script.yaml";
|
||||
await stat(metadataFilePath);
|
||||
const payload: any = await yamlParseFile(metadataFilePath);
|
||||
replaceLock(payload);
|
||||
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload,
|
||||
isJson: false,
|
||||
};
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function parseMetadataFile(
|
||||
scriptPath: string,
|
||||
generateMetadataIfMissing:
|
||||
@@ -770,62 +1022,87 @@ export async function parseMetadataFile(
|
||||
isJson: false,
|
||||
};
|
||||
} catch {
|
||||
// no metadata file at all. Create it
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Creating script metadata file for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
metadataFilePath = scriptPath + ".script.yaml";
|
||||
let scriptInitialMetadata = defaultScriptMetadata();
|
||||
const lockPath = scriptPath + ".script.lock";
|
||||
scriptInitialMetadata.lock = "!inline " + lockPath;
|
||||
const scriptInitialMetadataYaml = yamlStringify(
|
||||
scriptInitialMetadata as Record<string, any>,
|
||||
yamlOptions
|
||||
);
|
||||
|
||||
await writeFile(metadataFilePath, scriptInitialMetadataYaml, { flag: "wx", encoding: "utf-8" });
|
||||
await writeFile(lockPath, "", { flag: "wx", encoding: "utf-8" });
|
||||
|
||||
if (generateMetadataIfMissing) {
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Generating lockfile and schema for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
// Try folder layout: {scriptPath}__mod/script.yaml or .json
|
||||
const moduleFolderMeta = scriptPath + getModuleFolderSuffix();
|
||||
try {
|
||||
metadataFilePath = moduleFolderMeta + "/script.json";
|
||||
await stat(metadataFilePath);
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
|
||||
isJson: true,
|
||||
};
|
||||
} catch {
|
||||
try {
|
||||
await generateScriptMetadataInternal(
|
||||
generateMetadataIfMissing.path,
|
||||
generateMetadataIfMissing.workspaceRemote,
|
||||
generateMetadataIfMissing,
|
||||
false,
|
||||
false,
|
||||
generateMetadataIfMissing.rawWorkspaceDependencies,
|
||||
generateMetadataIfMissing.codebases,
|
||||
false
|
||||
);
|
||||
scriptInitialMetadata = (await yamlParseFile(
|
||||
metadataFilePath
|
||||
)) as ScriptMetadata;
|
||||
if (!generateMetadataIfMissing.schemaOnly) {
|
||||
replaceLock(scriptInitialMetadata);
|
||||
}
|
||||
} catch (e) {
|
||||
log.info(
|
||||
colors.yellow(
|
||||
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
|
||||
)
|
||||
);
|
||||
metadataFilePath = moduleFolderMeta + "/script.yaml";
|
||||
await stat(metadataFilePath);
|
||||
const payload: any = await yamlParseFile(metadataFilePath);
|
||||
replaceLock(payload);
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload,
|
||||
isJson: false,
|
||||
};
|
||||
} catch {
|
||||
// fall through to create metadata
|
||||
}
|
||||
}
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload: scriptInitialMetadata,
|
||||
isJson: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
// no metadata file at all. Create it
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Creating script metadata file for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
metadataFilePath = scriptPath + ".script.yaml";
|
||||
let scriptInitialMetadata = defaultScriptMetadata();
|
||||
const lockPath = scriptPath + ".script.lock";
|
||||
scriptInitialMetadata.lock = "!inline " + lockPath;
|
||||
const scriptInitialMetadataYaml = yamlStringify(
|
||||
scriptInitialMetadata as Record<string, any>,
|
||||
yamlOptions
|
||||
);
|
||||
|
||||
await writeFile(metadataFilePath, scriptInitialMetadataYaml, { flag: "wx", encoding: "utf-8" });
|
||||
await writeFile(lockPath, "", { flag: "wx", encoding: "utf-8" });
|
||||
|
||||
if (generateMetadataIfMissing) {
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Generating lockfile and schema for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
try {
|
||||
await generateScriptMetadataInternal(
|
||||
generateMetadataIfMissing.path,
|
||||
generateMetadataIfMissing.workspaceRemote,
|
||||
generateMetadataIfMissing,
|
||||
false,
|
||||
false,
|
||||
generateMetadataIfMissing.rawWorkspaceDependencies,
|
||||
generateMetadataIfMissing.codebases,
|
||||
false
|
||||
);
|
||||
scriptInitialMetadata = (await yamlParseFile(
|
||||
metadataFilePath
|
||||
)) as ScriptMetadata;
|
||||
if (!generateMetadataIfMissing.schemaOnly) {
|
||||
replaceLock(scriptInitialMetadata);
|
||||
}
|
||||
} catch (e) {
|
||||
log.info(
|
||||
colors.yellow(
|
||||
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload: scriptInitialMetadata,
|
||||
isJson: false,
|
||||
};
|
||||
}
|
||||
|
||||
interface Lock {
|
||||
@@ -834,6 +1111,7 @@ interface Lock {
|
||||
}
|
||||
|
||||
const WMILL_LOCKFILE = "wmill-lock.yaml";
|
||||
const SCRIPT_TOP_HASH = "__script_hash";
|
||||
|
||||
/**
|
||||
* Normalizes a path to use Linux separators (forward slashes).
|
||||
@@ -902,6 +1180,46 @@ export async function generateScriptHash(
|
||||
);
|
||||
}
|
||||
|
||||
async function computeModuleHashes(
|
||||
moduleFolderPath: string,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
isFolderLayout: boolean,
|
||||
): Promise<Record<string, string>> {
|
||||
const hashes: Record<string, string> = {};
|
||||
|
||||
async function readDir(dirPath: string, relPrefix: string) {
|
||||
const entries = readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
const isTopLevel = relPrefix === "";
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await readDir(fullPath, relPath);
|
||||
} else if (
|
||||
entry.isFile() &&
|
||||
!entry.name.endsWith(".lock") &&
|
||||
!(isFolderLayout && isTopLevel && entry.name.startsWith("script."))
|
||||
) {
|
||||
try {
|
||||
inferContentTypeFromFilePath(entry.name, defaultTs);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const content = readFileSync(fullPath, "utf-8");
|
||||
const normalizedPath = normalizeLockPath(relPath);
|
||||
hashes[normalizedPath] = await generateHash(
|
||||
content + JSON.stringify(rawWorkspaceDependencies)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await readDir(moduleFolderPath, "");
|
||||
return hashes;
|
||||
}
|
||||
|
||||
export async function clearGlobalLock(path: string): Promise<void> {
|
||||
const conf = await readLockfile();
|
||||
if (!conf?.locks) {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Relative Imports Utilities for CLI
|
||||
*
|
||||
* Provides functions to parse relative imports from TypeScript/Python scripts using WASM.
|
||||
*/
|
||||
|
||||
import { ScriptLanguage } from "./script_common.ts";
|
||||
import { loadParser } from "./metadata.ts";
|
||||
import * as log from "../core/log.ts";
|
||||
|
||||
/**
|
||||
* Extract relative imports from script content based on language.
|
||||
* Returns resolved absolute Windmill paths (e.g., "f/folder/helper").
|
||||
*/
|
||||
export async function extractRelativeImports(
|
||||
code: string,
|
||||
scriptPath: string,
|
||||
language: ScriptLanguage
|
||||
): Promise<string[]> {
|
||||
try {
|
||||
switch (language) {
|
||||
case "bun":
|
||||
case "nativets":
|
||||
case "deno": {
|
||||
const { parse_ts_relative_imports } = await loadParser("windmill-parser-wasm-ts");
|
||||
return parse_ts_relative_imports(code, scriptPath);
|
||||
}
|
||||
case "python3": {
|
||||
const { parse_py_relative_imports } = await loadParser("windmill-parser-wasm-py-imports");
|
||||
return parse_py_relative_imports(code, scriptPath);
|
||||
}
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn(`Failed to parse relative imports for ${scriptPath}: ${e}. Dependency tracking for relative imports will be disabled.`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -189,6 +189,26 @@ export function isFolderResourcePath(p: string): boolean {
|
||||
return isFlowPath(p) || isAppPath(p) || isRawAppPath(p);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path is inside a folder-based resource, checking BOTH dotted (.flow, .app, .raw_app)
|
||||
* and non-dotted (__flow, __app, __raw_app) formats regardless of the global nonDottedPaths setting.
|
||||
* Use this instead of isFolderResourcePath when the config may not yet be loaded or when
|
||||
* you need to handle mixed-format workspaces (e.g. generate-metadata scanning all files).
|
||||
*/
|
||||
export function isFolderResourcePathAnyFormat(p: string): boolean {
|
||||
const n = normalizeSep(p);
|
||||
for (const suffixes of [DOTTED_SUFFIXES, NON_DOTTED_SUFFIXES]) {
|
||||
if (
|
||||
n.includes(suffixes.flow + "/") ||
|
||||
n.includes(suffixes.app + "/") ||
|
||||
n.includes(suffixes.raw_app + "/")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect the resource type from a path, if any
|
||||
*/
|
||||
@@ -433,6 +453,66 @@ export function isRawAppFolderMetadataFile(p: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Script Module Path Functions
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* The suffix used for script module folders.
|
||||
* Unlike flows/apps, modules always use `__mod` (never dotted `.mod`)
|
||||
* to avoid confusion with file extensions.
|
||||
*/
|
||||
const MODULE_SUFFIX = "__mod";
|
||||
|
||||
/**
|
||||
* Get the module folder suffix (always "__mod")
|
||||
*/
|
||||
export function getModuleFolderSuffix(): string {
|
||||
return MODULE_SUFFIX;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path is inside a script module folder.
|
||||
* Matches patterns like: .../my_script__mod/...
|
||||
*/
|
||||
export function isScriptModulePath(p: string): boolean {
|
||||
return normalizeSep(p).includes(MODULE_SUFFIX + "/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the module folder path from a script's base path (without extension).
|
||||
* e.g., "f/my_script" -> "f/my_script__mod"
|
||||
*/
|
||||
export function buildModuleFolderPath(scriptBasePath: string): string {
|
||||
return scriptBasePath + MODULE_SUFFIX;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file inside a __mod/ folder is the main entry point (script.{ext}).
|
||||
* Entry points are files named "script.*" directly under __mod/ (not in subdirs).
|
||||
*/
|
||||
export function isModuleEntryPoint(p: string): boolean {
|
||||
const norm = normalizeSep(p);
|
||||
const suffix = MODULE_SUFFIX + "/";
|
||||
const idx = norm.indexOf(suffix);
|
||||
if (idx === -1) return false;
|
||||
const rest = norm.slice(idx + suffix.length);
|
||||
return rest.startsWith("script.") && !rest.includes("/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the script base path from a module folder entry.
|
||||
* e.g., "u/admin/my_script__mod/script.ts" -> "u/admin/my_script"
|
||||
* e.g., "u/admin/my_script__mod/helper.ts" -> "u/admin/my_script"
|
||||
*/
|
||||
export function getScriptBasePathFromModulePath(p: string): string | undefined {
|
||||
const norm = normalizeSep(p);
|
||||
const suffix = MODULE_SUFFIX + "/";
|
||||
const idx = norm.indexOf(suffix);
|
||||
if (idx === -1) return undefined;
|
||||
return norm.slice(0, idx);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Sync-related Path Functions
|
||||
// ============================================================================
|
||||
|
||||
@@ -30,13 +30,15 @@ export type WorkspaceDependenciesLanguage =
|
||||
| { language: "bun", filename /** (raw requirements filename) */: "package.json" }
|
||||
| { language: "python3", filename: "requirements.in" }
|
||||
| { language: "php", filename: "composer.json" }
|
||||
| { language: "go", filename: "go.mod" };
|
||||
| { language: "go", filename: "go.mod" }
|
||||
| { language: "powershell", filename: "modules.json" };
|
||||
|
||||
export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
|
||||
{ language: "bun", filename: "package.json" },
|
||||
{ language: "python3", filename: "requirements.in" },
|
||||
{ language: "php", filename: "composer.json" },
|
||||
{ language: "go", filename: "go.mod" },
|
||||
{ language: "powershell", filename: "modules.json" },
|
||||
] as const;
|
||||
|
||||
/**
|
||||
@@ -45,7 +47,8 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
|
||||
*/
|
||||
export function languageNeedsLock(language: ScriptLanguage | string): boolean {
|
||||
return (
|
||||
workspaceDependenciesLanguages.some((l) => l.language === language) ||
|
||||
(workspaceDependenciesLanguages.some((l) => l.language === language) &&
|
||||
language !== "powershell") ||
|
||||
language === "deno" ||
|
||||
language === "rust" ||
|
||||
language === "ansible"
|
||||
|
||||
@@ -63,11 +63,11 @@ export class CargoBackend {
|
||||
|
||||
// Determine default features based on environment
|
||||
// CI mode: minimal features (zip only)
|
||||
// Local mode with license key: full features (zip, private, enterprise, license)
|
||||
// Local mode with license key: full features (zip, private, enterprise, license, python)
|
||||
// Local mode without license key: zip only (EE features reject API calls without valid license)
|
||||
const isCI = process.env["CI_MINIMAL_FEATURES"] === "true";
|
||||
const hasLicenseKey = !!process.env["EE_LICENSE_KEY"];
|
||||
const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license"] : ["zip"]);
|
||||
const defaultFeatures = isCI ? ["zip"] : (hasLicenseKey ? ["zip", "private", "enterprise", "license", "python"] : ["zip", "python"]);
|
||||
|
||||
// Parse additional features from environment variable
|
||||
const envFeatures = process.env["TEST_FEATURES"]?.split(",").filter(f => f.trim()) || [];
|
||||
@@ -328,6 +328,8 @@ export class CargoBackend {
|
||||
SQLX_OFFLINE: "true",
|
||||
// Disable embedding to speed up startup
|
||||
DISABLE_EMBEDDING: "true",
|
||||
// Skip worker version check for workspace deps (workers need time to report version)
|
||||
WMDEBUG_FORCE_V0_WORKSPACE_DEPENDENCIES: "1",
|
||||
// Create default admin user
|
||||
CREATE_SUPERADMIN_IF_NOT_EXISTS: "1",
|
||||
SUPERADMIN_EMAIL: this.config.username,
|
||||
@@ -708,6 +710,7 @@ export class CargoBackend {
|
||||
this.deleteAll("resources"),
|
||||
this.deleteAll("variables"),
|
||||
this.deleteAll("folders"),
|
||||
this.deleteAllWorkspaceDeps(),
|
||||
]);
|
||||
|
||||
console.log("Workspace reset complete");
|
||||
@@ -735,6 +738,28 @@ export class CargoBackend {
|
||||
// Ignore listing failures
|
||||
}
|
||||
}
|
||||
|
||||
private async deleteAllWorkspaceDeps(): Promise<void> {
|
||||
try {
|
||||
const listResponse = await this.apiRequest(`/api/w/${this.config.workspace}/workspace_dependencies/list`);
|
||||
if (!listResponse.ok) return;
|
||||
|
||||
const items = await listResponse.json() as { language: string; name?: string }[];
|
||||
for (const item of items) {
|
||||
try {
|
||||
const nameParam = item.name ? `?name=${encodeURIComponent(item.name)}` : "";
|
||||
await this.apiRequest(
|
||||
`/api/w/${this.config.workspace}/workspace_dependencies/delete/${item.language}${nameParam}`,
|
||||
{ method: "POST" }
|
||||
);
|
||||
} catch {
|
||||
// Ignore individual deletion failures
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Ignore failures
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Global backend instance
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
/**
|
||||
* Datatable settings sync tests
|
||||
*
|
||||
* Tests that datatable config is correctly synced via settings.yaml during pull/push operations.
|
||||
*/
|
||||
|
||||
import { expect, test } from "bun:test";
|
||||
import { writeFile, readFile } from "node:fs/promises";
|
||||
import { parse, stringify } from "yaml";
|
||||
import { withTestBackend } from "./test_backend.ts";
|
||||
import { shouldSkipOnCI } from "./cargo_backend.ts";
|
||||
import { addWorkspace } from "../workspace.ts";
|
||||
|
||||
test.skipIf(shouldSkipOnCI())("Datatable config: included in sync pull settings.yaml", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const testWorkspace = {
|
||||
remote: backend.baseUrl,
|
||||
workspaceId: backend.workspace,
|
||||
name: "datatable_pull_test",
|
||||
token: backend.token
|
||||
};
|
||||
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
|
||||
|
||||
// Configure datatable on backend
|
||||
const datatableConfig = {
|
||||
datatables: {
|
||||
main: {
|
||||
database: {
|
||||
resource_path: "u/test/test_db",
|
||||
resource_type: "postgresql"
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const configResp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/workspaces/edit_datatable_config`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ settings: datatableConfig })
|
||||
}
|
||||
);
|
||||
expect(configResp.ok).toBe(true);
|
||||
|
||||
// Create wmill.yaml with includeSettings
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes:
|
||||
- "**"
|
||||
includeSettings: true`, "utf-8");
|
||||
|
||||
// Pull settings
|
||||
const result = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
|
||||
expect(result.code).toEqual(0);
|
||||
|
||||
// Verify settings.yaml was created and contains datatable
|
||||
const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8");
|
||||
expect(settingsContent).toContain("datatable:");
|
||||
expect(settingsContent).toContain("u/test/test_db");
|
||||
expect(settingsContent).toContain("postgresql");
|
||||
});
|
||||
});
|
||||
|
||||
test.skipIf(shouldSkipOnCI())("Datatable config: pushed correctly from settings.yaml", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const testWorkspace = {
|
||||
remote: backend.baseUrl,
|
||||
workspaceId: backend.workspace,
|
||||
name: "datatable_push_test",
|
||||
token: backend.token
|
||||
};
|
||||
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
|
||||
|
||||
// Create wmill.yaml with includeSettings
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes:
|
||||
- "**"
|
||||
includeSettings: true`, "utf-8");
|
||||
|
||||
// First pull to get baseline settings
|
||||
const pullResult = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Read existing settings and add datatable config
|
||||
const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8");
|
||||
const existingSettings = parse(settingsContent) as Record<string, unknown>;
|
||||
|
||||
existingSettings.datatable = {
|
||||
datatables: {
|
||||
analytics: {
|
||||
database: {
|
||||
resource_path: "u/admin/analytics_db",
|
||||
resource_type: "postgresql"
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await writeFile(`${tempDir}/settings.yaml`, stringify(existingSettings), "utf-8");
|
||||
|
||||
// Push the modified settings
|
||||
const pushResult = await backend.runCLICommand(['sync', 'push', '--yes'], tempDir);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
|
||||
// Verify the backend has the updated datatable config
|
||||
const settingsResp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/workspaces/get_settings`
|
||||
);
|
||||
expect(settingsResp.ok).toBe(true);
|
||||
const settings = await settingsResp.json();
|
||||
|
||||
expect(settings.datatable).toBeDefined();
|
||||
expect(settings.datatable.datatables).toBeDefined();
|
||||
expect(settings.datatable.datatables.analytics).toBeDefined();
|
||||
expect(settings.datatable.datatables.analytics.database.resource_path).toEqual("u/admin/analytics_db");
|
||||
expect(settings.datatable.datatables.analytics.database.resource_type).toEqual("postgresql");
|
||||
});
|
||||
});
|
||||
|
||||
test.skipIf(shouldSkipOnCI())("Datatable config: empty/undefined doesn't cause errors", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const testWorkspace = {
|
||||
remote: backend.baseUrl,
|
||||
workspaceId: backend.workspace,
|
||||
name: "datatable_empty_test",
|
||||
token: backend.token
|
||||
};
|
||||
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
|
||||
|
||||
// Ensure datatable config is empty/cleared on backend
|
||||
const clearResp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/workspaces/edit_datatable_config`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ settings: { datatables: {} } })
|
||||
}
|
||||
);
|
||||
expect(clearResp.ok).toBe(true);
|
||||
|
||||
// Create wmill.yaml with includeSettings
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes:
|
||||
- "**"
|
||||
includeSettings: true`, "utf-8");
|
||||
|
||||
// Pull should succeed even with empty datatable config
|
||||
const pullResult = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Settings.yaml should exist
|
||||
const settingsContent = await readFile(`${tempDir}/settings.yaml`, "utf-8");
|
||||
expect(settingsContent.length).toBeGreaterThan(0);
|
||||
|
||||
// Push should also succeed
|
||||
const pushResult = await backend.runCLICommand(['sync', 'push', '--yes'], tempDir);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
test.skipIf(shouldSkipOnCI())("Datatable config: round-trip preserves structure", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const testWorkspace = {
|
||||
remote: backend.baseUrl,
|
||||
workspaceId: backend.workspace,
|
||||
name: "datatable_roundtrip_test",
|
||||
token: backend.token
|
||||
};
|
||||
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
|
||||
|
||||
// Set up a complex datatable config on backend with multiple datatables
|
||||
const originalConfig = {
|
||||
datatables: {
|
||||
users: {
|
||||
database: {
|
||||
resource_path: "f/shared/users_db",
|
||||
resource_type: "postgresql"
|
||||
}
|
||||
},
|
||||
logs: {
|
||||
database: {
|
||||
resource_path: "f/shared/logs_db",
|
||||
resource_type: "instance"
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const configResp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/workspaces/edit_datatable_config`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ settings: originalConfig })
|
||||
}
|
||||
);
|
||||
expect(configResp.ok).toBe(true);
|
||||
|
||||
// Create wmill.yaml with includeSettings
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes:
|
||||
- "**"
|
||||
includeSettings: true`, "utf-8");
|
||||
|
||||
// Pull
|
||||
const pullResult = await backend.runCLICommand(['sync', 'pull', '--yes'], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Push back without modification
|
||||
const pushResult = await backend.runCLICommand(['sync', 'push', '--yes'], tempDir);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
|
||||
// Verify the config is preserved
|
||||
const settingsResp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/workspaces/get_settings`
|
||||
);
|
||||
expect(settingsResp.ok).toBe(true);
|
||||
const settings = await settingsResp.json();
|
||||
|
||||
expect(settings.datatable).toBeDefined();
|
||||
expect(settings.datatable.datatables.users).toBeDefined();
|
||||
expect(settings.datatable.datatables.logs).toBeDefined();
|
||||
expect(settings.datatable.datatables.users.database.resource_path).toEqual("f/shared/users_db");
|
||||
expect(settings.datatable.datatables.logs.database.resource_type).toEqual("instance");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,498 @@
|
||||
/**
|
||||
* Unit tests for failure_module and preprocessor_module inline script
|
||||
* extraction (pull) and replacement (push).
|
||||
*
|
||||
* These tests verify that rawscript content in failure_module and
|
||||
* preprocessor_module is correctly extracted to !inline references
|
||||
* and resolved back, matching the existing behavior for regular modules.
|
||||
*/
|
||||
|
||||
import { expect, test, describe } from "bun:test";
|
||||
import { extractInlineScripts, extractCurrentMapping } from "../windmill-utils-internal/src/inline-scripts/extractor.ts";
|
||||
import { replaceInlineScripts } from "../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { newPathAssigner } from "../windmill-utils-internal/src/path-utils/path-assigner.ts";
|
||||
import type { FlowModule } from "../windmill-utils-internal/src/gen/types.gen.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makeRawscriptModule(
|
||||
id: string,
|
||||
content: string,
|
||||
language: "bun" | "python3" | "deno" = "bun",
|
||||
lock?: string,
|
||||
): FlowModule {
|
||||
return {
|
||||
id,
|
||||
value: {
|
||||
type: "rawscript" as const,
|
||||
content,
|
||||
language,
|
||||
lock: lock,
|
||||
input_transforms: {},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const noopLogger = {
|
||||
info: () => {},
|
||||
error: () => {},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// extractInlineScripts — PULL direction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("extractInlineScripts for failure_module / preprocessor_module", () => {
|
||||
test("extracts rawscript from failure_module wrapped in array", () => {
|
||||
const failureModule = makeRawscriptModule(
|
||||
"failure",
|
||||
'export function main() { throw new Error("handler"); }',
|
||||
"bun",
|
||||
);
|
||||
|
||||
const scripts = extractInlineScripts([failureModule], {}, "/", "bun");
|
||||
|
||||
expect(scripts.length).toBeGreaterThanOrEqual(1);
|
||||
const script = scripts.find((s) => !s.is_lock);
|
||||
expect(script).toBeDefined();
|
||||
expect(script!.content).toBe(
|
||||
'export function main() { throw new Error("handler"); }',
|
||||
);
|
||||
// The module content should have been replaced with an !inline reference
|
||||
expect(failureModule.value.content).toStartWith("!inline ");
|
||||
});
|
||||
|
||||
test("extracts rawscript from preprocessor_module wrapped in array", () => {
|
||||
const preprocessorModule = makeRawscriptModule(
|
||||
"preprocessor",
|
||||
"export function main() { return {}; }",
|
||||
"python3",
|
||||
);
|
||||
|
||||
const scripts = extractInlineScripts(
|
||||
[preprocessorModule],
|
||||
{},
|
||||
"/",
|
||||
"bun",
|
||||
);
|
||||
|
||||
expect(scripts.length).toBeGreaterThanOrEqual(1);
|
||||
const script = scripts.find((s) => !s.is_lock);
|
||||
expect(script).toBeDefined();
|
||||
expect(script!.content).toBe("export function main() { return {}; }");
|
||||
expect(script!.language).toBe("python3");
|
||||
expect(preprocessorModule.value.content).toStartWith("!inline ");
|
||||
});
|
||||
|
||||
test("extracts lock alongside content", () => {
|
||||
const mod = makeRawscriptModule(
|
||||
"failure",
|
||||
"console.log('hi')",
|
||||
"bun",
|
||||
"some-lock-content",
|
||||
);
|
||||
|
||||
const scripts = extractInlineScripts([mod], {}, "/", "bun");
|
||||
|
||||
const contentScript = scripts.find((s) => !s.is_lock);
|
||||
const lockScript = scripts.find((s) => s.is_lock);
|
||||
expect(contentScript).toBeDefined();
|
||||
expect(lockScript).toBeDefined();
|
||||
expect(lockScript!.content).toBe("some-lock-content");
|
||||
expect((mod.value as any).lock).toStartWith("!inline ");
|
||||
});
|
||||
|
||||
test("shared pathAssigner prevents collisions when summaries match", () => {
|
||||
// If a regular module and failure_module share the same summary,
|
||||
// a shared PathAssigner deduplicates via its internal counter.
|
||||
const regular = makeRawscriptModule("a", "code_a", "bun");
|
||||
regular.summary = "my step";
|
||||
const failure = makeRawscriptModule("failure", "code_failure", "bun");
|
||||
failure.summary = "my step"; // same summary — would collide without shared assigner
|
||||
|
||||
const assigner = newPathAssigner("bun");
|
||||
const scripts1 = extractInlineScripts([regular], {}, "/", "bun", assigner);
|
||||
const scripts2 = extractInlineScripts([failure], {}, "/", "bun", assigner);
|
||||
|
||||
const allPaths = [...scripts1, ...scripts2]
|
||||
.filter((s) => !s.is_lock)
|
||||
.map((s) => s.path);
|
||||
|
||||
// All paths should be unique despite identical summaries
|
||||
expect(allPaths.length).toBe(2);
|
||||
expect(new Set(allPaths).size).toBe(2);
|
||||
});
|
||||
|
||||
test("without shared pathAssigner, identical summaries produce duplicate paths", () => {
|
||||
// Demonstrates the problem that sharing a PathAssigner solves.
|
||||
const regular = makeRawscriptModule("a", "code_a", "bun");
|
||||
regular.summary = "my step";
|
||||
const failure = makeRawscriptModule("failure", "code_failure", "bun");
|
||||
failure.summary = "my step";
|
||||
|
||||
// Separate assigners — each starts with a fresh counter
|
||||
const scripts1 = extractInlineScripts([regular], {}, "/", "bun");
|
||||
const scripts2 = extractInlineScripts([failure], {}, "/", "bun");
|
||||
|
||||
const allPaths = [...scripts1, ...scripts2]
|
||||
.filter((s) => !s.is_lock)
|
||||
.map((s) => s.path);
|
||||
|
||||
// Without a shared assigner, the paths collide
|
||||
expect(allPaths.length).toBe(2);
|
||||
expect(new Set(allPaths).size).toBe(1); // both got the same path
|
||||
});
|
||||
|
||||
test("skips non-rawscript failure_module (identity type)", () => {
|
||||
const identityModule: FlowModule = {
|
||||
id: "failure",
|
||||
value: { type: "identity" as any },
|
||||
};
|
||||
const scripts = extractInlineScripts([identityModule], {}, "/", "bun");
|
||||
expect(scripts).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// replaceInlineScripts — PUSH direction
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("replaceInlineScripts for failure_module / preprocessor_module", () => {
|
||||
test("resolves !inline reference back to file content", async () => {
|
||||
const failureModule = makeRawscriptModule(
|
||||
"failure",
|
||||
"!inline failure.inline_script.ts",
|
||||
"bun",
|
||||
);
|
||||
|
||||
const files: Record<string, string> = {
|
||||
"failure.inline_script.ts": 'export function main() { return "error handled"; }',
|
||||
};
|
||||
|
||||
await replaceInlineScripts(
|
||||
[failureModule],
|
||||
async (path) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(failureModule.value.content).toBe(
|
||||
'export function main() { return "error handled"; }',
|
||||
);
|
||||
});
|
||||
|
||||
test("resolves !inline reference for preprocessor_module", async () => {
|
||||
const preprocessorModule = makeRawscriptModule(
|
||||
"preprocessor",
|
||||
"!inline preprocessor.inline_script.py",
|
||||
"python3",
|
||||
);
|
||||
|
||||
const files: Record<string, string> = {
|
||||
"preprocessor.inline_script.py": "def main(): return {}",
|
||||
};
|
||||
|
||||
await replaceInlineScripts(
|
||||
[preprocessorModule],
|
||||
async (path) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(preprocessorModule.value.content).toBe("def main(): return {}");
|
||||
});
|
||||
|
||||
test("resolves !inline lock reference", async () => {
|
||||
const mod = makeRawscriptModule(
|
||||
"failure",
|
||||
"!inline failure.inline_script.ts",
|
||||
"bun",
|
||||
"!inline failure.inline_script.lock",
|
||||
);
|
||||
|
||||
const files: Record<string, string> = {
|
||||
"failure.inline_script.ts": "code here",
|
||||
"failure.inline_script.lock": "lock-data-here",
|
||||
};
|
||||
|
||||
await replaceInlineScripts(
|
||||
[mod],
|
||||
async (path) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(mod.value.content).toBe("code here");
|
||||
expect((mod.value as any).lock).toBe("lock-data-here");
|
||||
});
|
||||
|
||||
test("leaves non-inline content untouched", async () => {
|
||||
const mod = makeRawscriptModule(
|
||||
"failure",
|
||||
"export function main() { return 1; }",
|
||||
"bun",
|
||||
);
|
||||
|
||||
await replaceInlineScripts(
|
||||
[mod],
|
||||
async () => {
|
||||
throw new Error("should not be called");
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(mod.value.content).toBe("export function main() { return 1; }");
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Round-trip: extract then replace
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("round-trip extract → replace for failure_module / preprocessor_module", () => {
|
||||
test("failure_module content survives extract + replace", async () => {
|
||||
const originalContent = 'export function main(error: any) {\n console.error(error);\n return { handled: true };\n}';
|
||||
const failureModule = makeRawscriptModule(
|
||||
"failure",
|
||||
originalContent,
|
||||
"bun",
|
||||
);
|
||||
|
||||
// PULL: extract inline scripts (mutates module in place)
|
||||
const extracted = extractInlineScripts([failureModule], {}, "/", "bun");
|
||||
expect(failureModule.value.content).toStartWith("!inline ");
|
||||
|
||||
// Build a virtual filesystem from extracted scripts
|
||||
const files: Record<string, string> = {};
|
||||
for (const s of extracted) {
|
||||
files[s.path] = s.content;
|
||||
}
|
||||
|
||||
// PUSH: replace inline references back
|
||||
await replaceInlineScripts(
|
||||
[failureModule],
|
||||
async (path) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(failureModule.value.content).toBe(originalContent);
|
||||
});
|
||||
|
||||
test("preprocessor_module content survives extract + replace", async () => {
|
||||
const originalContent = "def main():\n return {\"preprocessed\": True}";
|
||||
const preprocessorModule = makeRawscriptModule(
|
||||
"preprocessor",
|
||||
originalContent,
|
||||
"python3",
|
||||
);
|
||||
|
||||
const extracted = extractInlineScripts(
|
||||
[preprocessorModule],
|
||||
{},
|
||||
"/",
|
||||
"bun",
|
||||
);
|
||||
expect(preprocessorModule.value.content).toStartWith("!inline ");
|
||||
|
||||
const files: Record<string, string> = {};
|
||||
for (const s of extracted) {
|
||||
files[s.path] = s.content;
|
||||
}
|
||||
|
||||
await replaceInlineScripts(
|
||||
[preprocessorModule],
|
||||
async (path) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(preprocessorModule.value.content).toBe(originalContent);
|
||||
});
|
||||
|
||||
test("failure_module with lock survives extract + replace", async () => {
|
||||
const originalContent = "export function main() { return 42; }";
|
||||
const originalLock = "package-lock-contents-here";
|
||||
const mod = makeRawscriptModule(
|
||||
"failure",
|
||||
originalContent,
|
||||
"bun",
|
||||
originalLock,
|
||||
);
|
||||
|
||||
const extracted = extractInlineScripts([mod], {}, "/", "bun");
|
||||
|
||||
const files: Record<string, string> = {};
|
||||
for (const s of extracted) {
|
||||
files[s.path] = s.content;
|
||||
}
|
||||
|
||||
await replaceInlineScripts(
|
||||
[mod],
|
||||
async (path) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
},
|
||||
noopLogger,
|
||||
"/tmp/test/",
|
||||
"/",
|
||||
);
|
||||
|
||||
expect(mod.value.content).toBe(originalContent);
|
||||
expect((mod.value as any).lock).toBe(originalLock);
|
||||
});
|
||||
|
||||
test("full flow with modules + failure_module + preprocessor_module round-trips", async () => {
|
||||
const regularContent = "export function main() { return 'step1'; }";
|
||||
const failureContent = "export function main(e: any) { return e; }";
|
||||
const preprocessorContent = "def main():\n pass";
|
||||
|
||||
const modules = [makeRawscriptModule("a", regularContent, "bun")];
|
||||
const failureModule = makeRawscriptModule("failure", failureContent, "bun");
|
||||
const preprocessorModule = makeRawscriptModule("preprocessor", preprocessorContent, "python3");
|
||||
|
||||
// Extract all (mimicking sync.ts pull logic)
|
||||
const allExtracted = [
|
||||
...extractInlineScripts(modules, {}, "/", "bun"),
|
||||
...extractInlineScripts([failureModule], {}, "/", "bun"),
|
||||
...extractInlineScripts([preprocessorModule], {}, "/", "bun"),
|
||||
];
|
||||
|
||||
// All modules should now have !inline references
|
||||
expect(modules[0].value.content).toStartWith("!inline ");
|
||||
expect(failureModule.value.content).toStartWith("!inline ");
|
||||
expect(preprocessorModule.value.content).toStartWith("!inline ");
|
||||
|
||||
// All paths should be unique
|
||||
const paths = allExtracted.filter((s) => !s.is_lock).map((s) => s.path);
|
||||
expect(new Set(paths).size).toBe(paths.length);
|
||||
|
||||
// Build filesystem
|
||||
const files: Record<string, string> = {};
|
||||
for (const s of allExtracted) {
|
||||
files[s.path] = s.content;
|
||||
}
|
||||
|
||||
const fileReader = async (path: string) => {
|
||||
if (!(path in files)) throw new Error(`File not found: ${path}`);
|
||||
return files[path];
|
||||
};
|
||||
|
||||
// Replace all (mimicking flow.ts push logic)
|
||||
await replaceInlineScripts(modules, fileReader, noopLogger, "/tmp/", "/");
|
||||
await replaceInlineScripts([failureModule], fileReader, noopLogger, "/tmp/", "/");
|
||||
await replaceInlineScripts([preprocessorModule], fileReader, noopLogger, "/tmp/", "/");
|
||||
|
||||
expect(modules[0].value.content).toBe(regularContent);
|
||||
expect(failureModule.value.content).toBe(failureContent);
|
||||
expect(preprocessorModule.value.content).toBe(preprocessorContent);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// extractCurrentMapping
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("extractCurrentMapping for failure_module / preprocessor_module", () => {
|
||||
test("extracts mapping from failure_module via optional param", () => {
|
||||
const failureModule: FlowModule = makeRawscriptModule(
|
||||
"failure",
|
||||
"!inline failure.inline_script.ts",
|
||||
"bun",
|
||||
);
|
||||
|
||||
const mapping = extractCurrentMapping(
|
||||
undefined,
|
||||
{},
|
||||
failureModule,
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(mapping["failure"]).toBe("failure.inline_script.ts");
|
||||
});
|
||||
|
||||
test("extracts mapping from preprocessor_module via optional param", () => {
|
||||
const preprocessorModule: FlowModule = makeRawscriptModule(
|
||||
"preprocessor",
|
||||
"!inline preprocessor.inline_script.py",
|
||||
"python3",
|
||||
);
|
||||
|
||||
const mapping = extractCurrentMapping(
|
||||
undefined,
|
||||
{},
|
||||
undefined,
|
||||
preprocessorModule,
|
||||
);
|
||||
|
||||
expect(mapping["preprocessor"]).toBe("preprocessor.inline_script.py");
|
||||
});
|
||||
|
||||
test("extracts mapping from modules + failure + preprocessor combined", () => {
|
||||
const modules: FlowModule[] = [
|
||||
makeRawscriptModule("a", "!inline a.inline_script.ts", "bun"),
|
||||
];
|
||||
const failureModule = makeRawscriptModule(
|
||||
"failure",
|
||||
"!inline failure.inline_script.ts",
|
||||
"bun",
|
||||
);
|
||||
const preprocessorModule = makeRawscriptModule(
|
||||
"preprocessor",
|
||||
"!inline preprocessor.inline_script.py",
|
||||
"python3",
|
||||
);
|
||||
|
||||
const mapping = extractCurrentMapping(
|
||||
modules,
|
||||
{},
|
||||
failureModule,
|
||||
preprocessorModule,
|
||||
);
|
||||
|
||||
expect(mapping["a"]).toBe("a.inline_script.ts");
|
||||
expect(mapping["failure"]).toBe("failure.inline_script.ts");
|
||||
expect(mapping["preprocessor"]).toBe("preprocessor.inline_script.py");
|
||||
});
|
||||
|
||||
test("ignores non-inline content in failure_module", () => {
|
||||
const failureModule = makeRawscriptModule(
|
||||
"failure",
|
||||
"export function main() {}",
|
||||
"bun",
|
||||
);
|
||||
|
||||
const mapping = extractCurrentMapping(
|
||||
undefined,
|
||||
{},
|
||||
failureModule,
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(mapping["failure"]).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,7 @@
|
||||
import { expect, test } from "bun:test";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import { withTestBackend } from "./test_backend.ts";
|
||||
import { shouldSkipOnCI } from "./cargo_backend.ts";
|
||||
|
||||
// =============================================================================
|
||||
// PREVIEW COMMAND INTEGRATION TESTS
|
||||
@@ -27,6 +28,7 @@ async function createWmillConfig(
|
||||
relative_path: string;
|
||||
includes?: string[];
|
||||
format?: "cjs" | "esm";
|
||||
customBundler?: string;
|
||||
assets?: Array<{ from: string; to: string }>;
|
||||
}>;
|
||||
}
|
||||
@@ -44,6 +46,9 @@ async function createWmillConfig(
|
||||
if (cb.format) {
|
||||
yamlContent += ` format: ${cb.format}\n`;
|
||||
}
|
||||
if (cb.customBundler) {
|
||||
yamlContent += ` customBundler: ${JSON.stringify(cb.customBundler)}\n`;
|
||||
}
|
||||
if (cb.assets && cb.assets.length > 0) {
|
||||
yamlContent += " assets:\n";
|
||||
for (const asset of cb.assets) {
|
||||
@@ -122,6 +127,39 @@ schema:
|
||||
await writeFile(`${dir}/flow.yaml`, flowYaml, "utf-8");
|
||||
}
|
||||
|
||||
async function createPathScriptFlow(
|
||||
tempDir: string,
|
||||
flowPath: string,
|
||||
options: {
|
||||
summary: string;
|
||||
scriptPath: string;
|
||||
inputTransforms?: string;
|
||||
}
|
||||
): Promise<void> {
|
||||
const dir = `${tempDir}/${flowPath}`;
|
||||
await mkdir(dir, { recursive: true });
|
||||
const inputTransforms = options.inputTransforms
|
||||
? ` input_transforms:\n${options.inputTransforms}`
|
||||
: " input_transforms: {}\n";
|
||||
|
||||
const flowYaml = `summary: "${options.summary}"
|
||||
description: "Test flow"
|
||||
value:
|
||||
modules:
|
||||
- id: "a"
|
||||
value:
|
||||
type: "script"
|
||||
path: "${options.scriptPath}"
|
||||
${inputTransforms}
|
||||
schema:
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema"
|
||||
type: object
|
||||
properties: {}
|
||||
required: []
|
||||
`;
|
||||
await writeFile(`${dir}/flow.yaml`, flowYaml, "utf-8");
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// SCRIPT PREVIEW TESTS
|
||||
// =============================================================================
|
||||
@@ -359,6 +397,116 @@ schema:
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// SCRIPT WITH MODULES PREVIEW TESTS
|
||||
// =============================================================================
|
||||
|
||||
test("script preview: script with modules (taskScript pattern)", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "bun" });
|
||||
|
||||
// Create the main script that uses taskScript to call a module
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/test/wac_script.ts",
|
||||
`import { task, taskScript, workflow } from "windmill-client";
|
||||
|
||||
const helper = taskScript("./helper.ts");
|
||||
|
||||
const process = task(async (x: string): Promise<string> => {
|
||||
return \`processed: \${x}\`;
|
||||
});
|
||||
|
||||
export const main = workflow(async (x: string = "test") => {
|
||||
const a = await process(x);
|
||||
const b = await helper({ a });
|
||||
return { processed: a, helper_result: b };
|
||||
});`
|
||||
);
|
||||
|
||||
// Create the module file in __mod/ folder
|
||||
const modDir = `${tempDir}/f/test/wac_script__mod`;
|
||||
await mkdir(modDir, { recursive: true });
|
||||
await writeFile(
|
||||
`${modDir}/helper.ts`,
|
||||
`export function main(a: string): string {
|
||||
return \`helper got: \${a}\`;
|
||||
}`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["script", "preview", "f/test/wac_script.ts"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("processed: test");
|
||||
expect(output).toContain("helper got:");
|
||||
});
|
||||
});
|
||||
|
||||
test("script preview: script with modules (folder layout)", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "bun" });
|
||||
|
||||
// Create folder layout: my_script__mod/script.ts + my_script__mod/helper.ts
|
||||
const modDir = `${tempDir}/f/test/folder_wac__mod`;
|
||||
await mkdir(modDir, { recursive: true });
|
||||
|
||||
// Entry point script
|
||||
await writeFile(
|
||||
`${modDir}/script.ts`,
|
||||
`import { task, taskScript, workflow } from "windmill-client";
|
||||
|
||||
const helper = taskScript("./helper.ts");
|
||||
|
||||
export const main = workflow(async (name: string = "World") => {
|
||||
const result = await helper({ name });
|
||||
return { greeting: result };
|
||||
});`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Module file
|
||||
await writeFile(
|
||||
`${modDir}/helper.ts`,
|
||||
`export function main(name: string): string {
|
||||
return \`Hello from module, \${name}!\`;
|
||||
}`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Script metadata
|
||||
await writeFile(
|
||||
`${modDir}/script.yaml`,
|
||||
`summary: "Folder layout WAC script"
|
||||
description: "Test"
|
||||
lock: ""
|
||||
schema:
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema"
|
||||
type: object
|
||||
properties:
|
||||
name:
|
||||
type: string
|
||||
default: "World"
|
||||
required: []
|
||||
`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["script", "preview", `f/test/folder_wac__mod/script.ts`],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("Hello from module, World!");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// FLOW PREVIEW TESTS
|
||||
// =============================================================================
|
||||
@@ -381,3 +529,280 @@ test("flow preview: simple flow", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("flow preview: uses local PathScript by default and remote PathScript with --remote", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "bun" });
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/test/helper_script.ts",
|
||||
`export function main(name: string = "World") { return \`Remote script says: \${name}!\`; }`
|
||||
);
|
||||
|
||||
const pushResult = await backend.runCLICommand(
|
||||
["script", "push", "f/test/helper_script.ts"],
|
||||
tempDir
|
||||
);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
|
||||
await writeFile(
|
||||
`${tempDir}/f/test/helper_script.ts`,
|
||||
`export function main(name: string = "World") { return \`Local script says: \${name}!\`; }`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/path_flow.flow", {
|
||||
summary: "Flow with PathScript",
|
||||
scriptPath: "f/test/helper_script",
|
||||
inputTransforms: ` name:
|
||||
type: "static"
|
||||
value: "PathTest"
|
||||
`,
|
||||
});
|
||||
|
||||
const localResult = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(localResult.code).toEqual(0);
|
||||
expect(localResult.stdout + localResult.stderr).toContain(
|
||||
"Local script says: PathTest!"
|
||||
);
|
||||
expect(localResult.stdout + localResult.stderr).toContain(
|
||||
"Using local PathScript files for flow preview."
|
||||
);
|
||||
expect(localResult.stdout + localResult.stderr).toContain(
|
||||
"These workspace scripts differ from the deployed version:\n- f/test/helper_script"
|
||||
);
|
||||
|
||||
const remoteResult = await backend.runCLICommand(
|
||||
["flow", "preview", "--remote", "f/test/path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(remoteResult.code).toEqual(0);
|
||||
expect(remoteResult.stdout + remoteResult.stderr).toContain(
|
||||
"Remote script says: PathTest!"
|
||||
);
|
||||
expect(remoteResult.stdout + remoteResult.stderr).not.toContain(
|
||||
"Using local PathScript files for flow preview."
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test.skipIf(shouldSkipOnCI())("flow preview: respects defaultTs when resolving local PathScripts", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "deno" });
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/test/deno_helper.ts",
|
||||
`export function main() { return Deno.version.deno ? "deno-runtime" : "missing"; }`
|
||||
);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/deno_path_flow.flow", {
|
||||
summary: "Flow with Deno PathScript",
|
||||
scriptPath: "f/test/deno_helper",
|
||||
});
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/deno_path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout + result.stderr).toContain("deno-runtime");
|
||||
});
|
||||
});
|
||||
|
||||
test("flow preview: bundles local PathScripts with local imports", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, {
|
||||
defaultTs: "bun",
|
||||
codebases: [{ relative_path: "f/flow_codebase", includes: ["**"] }],
|
||||
});
|
||||
|
||||
await mkdir(`${tempDir}/f/flow_codebase`, { recursive: true });
|
||||
await writeFile(
|
||||
`${tempDir}/f/flow_codebase/helper.ts`,
|
||||
`export function greet(name: string): string {
|
||||
return \`Hello from local flow codebase, \${name}!\`;
|
||||
}`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/flow_codebase/main_script.ts",
|
||||
`import { greet } from "./helper";
|
||||
|
||||
export function main(name: string = "World") {
|
||||
return greet(name);
|
||||
}`
|
||||
);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/importing_path_flow.flow", {
|
||||
summary: "Flow with imported PathScript",
|
||||
scriptPath: "f/flow_codebase/main_script",
|
||||
inputTransforms: ` name:
|
||||
type: "static"
|
||||
value: "FlowTest"
|
||||
`,
|
||||
});
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/importing_path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout + result.stderr).toContain(
|
||||
"Hello from local flow codebase, FlowTest!"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("flow preview: customBundler handles script paths with spaces", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, {
|
||||
defaultTs: "bun",
|
||||
codebases: [{
|
||||
relative_path: "f/codebase custom",
|
||||
includes: ["f/codebase custom/**"],
|
||||
customBundler: "cat",
|
||||
}],
|
||||
});
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/codebase custom/custom bundler.ts",
|
||||
`export function main() {
|
||||
return "Custom bundler path with spaces";
|
||||
}`
|
||||
);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/custom_bundler_path.flow", {
|
||||
summary: "Flow with customBundler path",
|
||||
scriptPath: "f/codebase custom/custom bundler",
|
||||
});
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/custom_bundler_path.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout + result.stderr).toContain(
|
||||
"Custom bundler path with spaces"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("flow preview: warns when local PathScript is not deployed remotely", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "bun" });
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/test/undeployed_helper.ts",
|
||||
`export function main() { return "Local only script"; }`
|
||||
);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/undeployed_path_flow.flow", {
|
||||
summary: "Flow with undeployed PathScript",
|
||||
scriptPath: "f/test/undeployed_helper",
|
||||
});
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/undeployed_path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout + result.stderr).toContain("Local only script");
|
||||
expect(result.stdout + result.stderr).toContain(
|
||||
"These scripts do not exist in the workspace yet:\n- f/test/undeployed_helper"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("flow preview: does not warn when local and deployed PathScripts match", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, { defaultTs: "bun" });
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/test/matching_helper.ts",
|
||||
`export function main() { return "Matching script"; }`
|
||||
);
|
||||
|
||||
const pushResult = await backend.runCLICommand(
|
||||
["script", "push", "f/test/matching_helper.ts"],
|
||||
tempDir
|
||||
);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/matching_path_flow.flow", {
|
||||
summary: "Flow with matching PathScript",
|
||||
scriptPath: "f/test/matching_helper",
|
||||
});
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/matching_path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout + result.stderr).toContain("Matching script");
|
||||
expect(result.stdout + result.stderr).not.toContain(
|
||||
"Using local PathScript files for flow preview."
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
test("flow preview: fails loudly for asset-backed codebase scripts", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await createWmillConfig(tempDir, {
|
||||
defaultTs: "bun",
|
||||
codebases: [{
|
||||
relative_path: "f/codebase_tar",
|
||||
includes: ["**"],
|
||||
assets: [{ from: "f/codebase_tar/data.json", to: "data.json" }],
|
||||
}],
|
||||
});
|
||||
|
||||
await mkdir(`${tempDir}/f/codebase_tar`, { recursive: true });
|
||||
await writeFile(
|
||||
`${tempDir}/f/codebase_tar/data.json`,
|
||||
JSON.stringify({ message: "Hello from asset!" }),
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
await createScript(
|
||||
tempDir,
|
||||
"f/codebase_tar/main_script.ts",
|
||||
`import * as fs from "fs";
|
||||
|
||||
export function main() {
|
||||
const data = JSON.parse(fs.readFileSync("data.json", "utf-8"));
|
||||
return data.message;
|
||||
}`
|
||||
);
|
||||
|
||||
await createPathScriptFlow(tempDir, "f/test/assets_path_flow.flow", {
|
||||
summary: "Flow with asset-backed PathScript",
|
||||
scriptPath: "f/codebase_tar/main_script",
|
||||
});
|
||||
|
||||
const localResult = await backend.runCLICommand(
|
||||
["flow", "preview", "f/test/assets_path_flow.flow"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(localResult.code).not.toEqual(0);
|
||||
expect(localResult.stdout + localResult.stderr).toContain(
|
||||
"requires codebase assets"
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -107,7 +107,7 @@ async function readFileContent(filePath: string): Promise<string> {
|
||||
* Create a raw app directory structure on disk
|
||||
* Uses .raw_app folder suffix with raw_app.yaml metadata
|
||||
*/
|
||||
async function createRawAppOnDisk(appDir: string): Promise<void> {
|
||||
async function createRawAppOnDisk(appDir: string, includeBackend: boolean = false): Promise<void> {
|
||||
await mkdir(appDir, { recursive: true });
|
||||
await mkdir(path.join(appDir, "inline_scripts"), { recursive: true });
|
||||
|
||||
@@ -131,6 +131,16 @@ async function createRawAppOnDisk(appDir: string): Promise<void> {
|
||||
INLINE_SCRIPT_A_LOCK,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Optionally create backend runnable (type: inline)
|
||||
if (includeBackend) {
|
||||
await mkdir(path.join(appDir, "backend"), { recursive: true });
|
||||
await writeFile(path.join(appDir, "backend", "query.yaml"), "type: inline\n", "utf-8");
|
||||
await writeFile(path.join(appDir, "backend", "query.ts"), `export async function main(x: number): Promise<string> {
|
||||
return \`Result: \${x}\`;
|
||||
}
|
||||
`, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
test("Raw App: full sync workflow - push, pull, modify, push, clear, pull", async () => {
|
||||
@@ -153,7 +163,7 @@ excludes: []`, "utf-8");
|
||||
// Create folder structure
|
||||
const appDir = path.join(tempDir, "f", "test", "my_raw_app.raw_app");
|
||||
await mkdir(path.join(tempDir, "f", "test"), { recursive: true });
|
||||
await createRawAppOnDisk(appDir);
|
||||
await createRawAppOnDisk(appDir, true); // Include backend for metadata test
|
||||
|
||||
// =========================================================================
|
||||
// STEP 1: Initial push - create raw app on backend
|
||||
@@ -266,6 +276,47 @@ excludes: []`, "utf-8");
|
||||
|
||||
const pulledInlineScript = await readFileContent(inlineScriptPath);
|
||||
expect(pulledInlineScript).toContain("modified:");
|
||||
|
||||
// =========================================================================
|
||||
// STEP 7: Test that script generate-metadata does NOT process backend runnables
|
||||
// =========================================================================
|
||||
|
||||
// Create a standalone script (should be processed by script generate-metadata)
|
||||
await writeFile(path.join(tempDir, "f", "test", "standalone.ts"), `export async function main(): Promise<string> {
|
||||
return "hello";
|
||||
}
|
||||
`, "utf-8");
|
||||
|
||||
// Run script generate-metadata
|
||||
const metaResult1 = await backend.runCLICommand(
|
||||
['script', 'generate-metadata', '--yes'],
|
||||
tempDir, "raw_app_test"
|
||||
);
|
||||
expect(metaResult1.code).toEqual(0);
|
||||
|
||||
// Run generate-metadata --skip-flows --skip-apps
|
||||
const metaResult2 = await backend.runCLICommand(
|
||||
['generate-metadata', '--skip-flows', '--skip-apps', '--yes'],
|
||||
tempDir, "raw_app_test"
|
||||
);
|
||||
expect(metaResult2.code).toEqual(0);
|
||||
|
||||
// Backend runnables should NOT have .script.yaml files
|
||||
const backendDir = path.join(appDir, "backend");
|
||||
expect(await fileExists(path.join(backendDir, "query.yaml"))).toBeTruthy();
|
||||
expect(await fileExists(path.join(backendDir, "query.ts"))).toBeTruthy();
|
||||
expect(await fileExists(path.join(backendDir, "query.script.yaml"))).toBeFalsy();
|
||||
expect(await fileExists(path.join(backendDir, "query.script.lock"))).toBeFalsy();
|
||||
|
||||
// Bug: raw app backend files get misprocessed and create script files at wrong location
|
||||
// The path f/test/my_raw_app.raw_app/backend/query.ts gets truncated at first "."
|
||||
// becoming f/test/my_raw_app.script.yaml (stripping .raw_app/backend/query.ts)
|
||||
expect(await fileExists(path.join(tempDir, "f", "test", "my_raw_app.script.yaml"))).toBeFalsy();
|
||||
expect(await fileExists(path.join(tempDir, "f", "test", "my_raw_app.script.lock"))).toBeFalsy();
|
||||
|
||||
// Standalone script SHOULD have metadata
|
||||
expect(await fileExists(path.join(tempDir, "f", "test", "standalone.script.yaml"))).toBeTruthy();
|
||||
expect(await fileExists(path.join(tempDir, "f", "test", "standalone.script.lock"))).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,420 @@
|
||||
/**
|
||||
* Relative Imports Tests
|
||||
*
|
||||
* E2E tests for the `generate-metadata` command with relative imports:
|
||||
* - Lock files correctly include transitive dependencies
|
||||
* - Staleness propagates through import chains
|
||||
* - Various import patterns handled correctly
|
||||
*/
|
||||
|
||||
import { expect, test } from "bun:test";
|
||||
import { writeFile, readFile, mkdir } from "node:fs/promises";
|
||||
import { withTestBackend } from "./test_backend.ts";
|
||||
|
||||
// TODO: re-enable Python tests on CI if python feature is included by default
|
||||
const isCI = process.env["CI_MINIMAL_FEATURES"] === "true";
|
||||
|
||||
const defaultMetadata = `summary: "Test"
|
||||
schema:
|
||||
type: object
|
||||
properties: {}
|
||||
lock: ""
|
||||
`;
|
||||
|
||||
// =============================================================================
|
||||
// Test 1: TS basic import with npm dependency propagation
|
||||
// =============================================================================
|
||||
|
||||
test("TS: imported script's npm dep appears in importer's lock", { timeout: 60000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
const scriptA = `import { helper } from "./script_b.ts";
|
||||
export async function main() { return helper(); }
|
||||
`;
|
||||
const scriptB = `import _ from "lodash";
|
||||
export function helper() { return _.VERSION; }
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
|
||||
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
|
||||
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(result.code, `generate-metadata failed:\nSTDOUT: ${result.stdout}\nSTDERR: ${result.stderr}`).toBe(0);
|
||||
|
||||
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
|
||||
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockB).toContain("lodash");
|
||||
expect(lockA).toContain("lodash");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 2: TS chained imports - dependency propagates through chain
|
||||
// =============================================================================
|
||||
|
||||
test("TS: chained imports propagate npm deps through entire chain", { timeout: 60000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
const scriptA = `import { utilB } from "./script_b.ts";
|
||||
export async function main() { return utilB(); }
|
||||
`;
|
||||
const scriptB = `import { utilC } from "./script_c.ts";
|
||||
export function utilB() { return utilC() + " B"; }
|
||||
`;
|
||||
const scriptC = `import _ from "lodash";
|
||||
export function utilC() { return _.VERSION; }
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
|
||||
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
|
||||
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC);
|
||||
await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(result.code).toBe(0);
|
||||
|
||||
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
|
||||
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
|
||||
const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockC).toContain("lodash");
|
||||
expect(lockB).toContain("lodash");
|
||||
expect(lockA).toContain("lodash");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 3: TS circular imports - completes without hanging, locks generated
|
||||
// =============================================================================
|
||||
|
||||
test("TS: circular imports handled gracefully with correct locks", { timeout: 60000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
// Circular: A imports B, B imports A, B has npm dep
|
||||
const scriptA = `import { funcB } from "./script_b.ts";
|
||||
export function funcA() { return "A"; }
|
||||
export async function main() { return funcA() + funcB(); }
|
||||
`;
|
||||
const scriptB = `import { funcA } from "./script_a.ts";
|
||||
import _ from "lodash";
|
||||
export function funcB() { return _.VERSION + funcA(); }
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
|
||||
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
|
||||
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(result.code).toBe(0);
|
||||
|
||||
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
|
||||
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockB).toContain("lodash");
|
||||
expect(lockA).toContain("lodash");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 4: Python basic import with pip dependency propagation
|
||||
// =============================================================================
|
||||
|
||||
test.skipIf(isCI)("Python: imported script's pip dep appears in importer's lock", { timeout: 60000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
const mainPy = `from f.test.helper import helper_func
|
||||
|
||||
def main():
|
||||
return helper_func()
|
||||
`;
|
||||
const helperPy = `import requests
|
||||
|
||||
def helper_func():
|
||||
return requests.__version__
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/main.py`, mainPy);
|
||||
await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/helper.py`, helperPy);
|
||||
await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
if (result.code !== 0) {
|
||||
console.log("STDOUT:", result.stdout);
|
||||
console.log("STDERR:", result.stderr);
|
||||
}
|
||||
expect(result.code).toBe(0);
|
||||
|
||||
const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => "");
|
||||
const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockHelper).toContain("requests");
|
||||
expect(lockMain).toContain("requests");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 5: Diamond dependency - A imports B and C, both import D
|
||||
// =============================================================================
|
||||
|
||||
test.skipIf(isCI)("Python: diamond dependency pattern propagates correctly", { timeout: 60000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
// Diamond: A -> B, A -> C, B -> D, C -> D
|
||||
const scriptA = `from f.test.script_b import func_b
|
||||
from f.test.script_c import func_c
|
||||
|
||||
def main():
|
||||
return func_b() + func_c()
|
||||
`;
|
||||
const scriptB = `from f.test.script_d import func_d
|
||||
|
||||
def func_b():
|
||||
return "B" + func_d()
|
||||
`;
|
||||
const scriptC = `from f.test.script_d import func_d
|
||||
|
||||
def func_c():
|
||||
return "C" + func_d()
|
||||
`;
|
||||
const scriptD = `import requests
|
||||
|
||||
def func_d():
|
||||
return requests.__version__
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/script_a.py`, scriptA);
|
||||
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_b.py`, scriptB);
|
||||
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_c.py`, scriptC);
|
||||
await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_d.py`, scriptD);
|
||||
await writeFile(`${tempDir}/f/test/script_d.script.yaml`, defaultMetadata);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(result.code).toBe(0);
|
||||
|
||||
const lockA = await readFile(`${tempDir}/f/test/script_a.script.lock`, "utf-8").catch(() => "");
|
||||
const lockB = await readFile(`${tempDir}/f/test/script_b.script.lock`, "utf-8").catch(() => "");
|
||||
const lockC = await readFile(`${tempDir}/f/test/script_c.script.lock`, "utf-8").catch(() => "");
|
||||
const lockD = await readFile(`${tempDir}/f/test/script_d.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockD).toContain("requests");
|
||||
expect(lockB).toContain("requests");
|
||||
expect(lockC).toContain("requests");
|
||||
expect(lockA).toContain("requests");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 6: Script isolation - unrelated script not marked stale
|
||||
// =============================================================================
|
||||
|
||||
test("Script isolation: unrelated script not affected by changes", { timeout: 120000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
// A imports B, C is isolated
|
||||
const scriptA = `import { helper } from "./script_b.ts";
|
||||
export async function main() { return helper(); }
|
||||
`;
|
||||
const scriptB = `export function helper() { return "B"; }
|
||||
`;
|
||||
const scriptC = `export async function main() { return "isolated"; }
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/script_a.ts`, scriptA);
|
||||
await writeFile(`${tempDir}/f/test/script_a.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_b.ts`, scriptB);
|
||||
await writeFile(`${tempDir}/f/test/script_b.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/script_c.ts`, scriptC);
|
||||
await writeFile(`${tempDir}/f/test/script_c.script.yaml`, defaultMetadata);
|
||||
|
||||
// Generate initial metadata
|
||||
const initial = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(initial.code).toBe(0);
|
||||
|
||||
// Verify all up to date
|
||||
const check1 = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"],
|
||||
tempDir
|
||||
);
|
||||
expect(check1.stdout).toContain("All metadata up-to-date");
|
||||
|
||||
// Change script_b
|
||||
await writeFile(`${tempDir}/f/test/script_b.ts`,
|
||||
`export function helper() { return "B changed"; }
|
||||
`);
|
||||
|
||||
// script_a and script_b should be stale, script_c should NOT be mentioned
|
||||
const check2 = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes", "--dry-run"],
|
||||
tempDir
|
||||
);
|
||||
expect(check2.code).toBe(0);
|
||||
expect(check2.stdout).toContain("script_b");
|
||||
expect(check2.stdout).toContain("script_a");
|
||||
expect(check2.stdout).not.toMatch(/script_c/);
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 7: Python relative imports with dot syntax
|
||||
// =============================================================================
|
||||
|
||||
test.skipIf(isCI)("Python: relative imports with dot syntax work correctly", { timeout: 60000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/mymodule`, { recursive: true });
|
||||
|
||||
// Using relative import syntax
|
||||
const mainPy = `from .helper import helper_func
|
||||
|
||||
def main():
|
||||
return helper_func()
|
||||
`;
|
||||
const helperPy = `import requests
|
||||
|
||||
def helper_func():
|
||||
return requests.__version__
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/mymodule/main.py`, mainPy);
|
||||
await writeFile(`${tempDir}/f/mymodule/main.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/mymodule/helper.py`, helperPy);
|
||||
await writeFile(`${tempDir}/f/mymodule/helper.script.yaml`, defaultMetadata);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/mymodule/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(result.code).toBe(0);
|
||||
|
||||
const lockMain = await readFile(`${tempDir}/f/mymodule/main.script.lock`, "utf-8").catch(() => "");
|
||||
const lockHelper = await readFile(`${tempDir}/f/mymodule/helper.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockHelper).toContain("requests");
|
||||
expect(lockMain).toContain("requests");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Test 8: Adding new import updates importer's lock
|
||||
// =============================================================================
|
||||
|
||||
test.skipIf(isCI)("Python: adding new import updates importer's lock correctly", { timeout: 120000 }, async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `includes: ["**"]
|
||||
excludes: []`);
|
||||
|
||||
await mkdir(`${tempDir}/f/test`, { recursive: true });
|
||||
|
||||
// Initial: main imports helper, helper has no external deps
|
||||
const mainPy = `from f.test.helper import helper_func
|
||||
|
||||
def main():
|
||||
return helper_func()
|
||||
`;
|
||||
const helperPyInitial = `def helper_func():
|
||||
return "no deps"
|
||||
`;
|
||||
|
||||
await writeFile(`${tempDir}/f/test/main.py`, mainPy);
|
||||
await writeFile(`${tempDir}/f/test/main.script.yaml`, defaultMetadata);
|
||||
await writeFile(`${tempDir}/f/test/helper.py`, helperPyInitial);
|
||||
await writeFile(`${tempDir}/f/test/helper.script.yaml`, defaultMetadata);
|
||||
|
||||
// Generate initial locks
|
||||
const initial = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(initial.code).toBe(0);
|
||||
|
||||
// Add new script with pip dep
|
||||
const utilsPy = `import requests
|
||||
|
||||
def get_version():
|
||||
return requests.__version__
|
||||
`;
|
||||
await writeFile(`${tempDir}/f/test/utils.py`, utilsPy);
|
||||
await writeFile(`${tempDir}/f/test/utils.script.yaml`, defaultMetadata);
|
||||
|
||||
// Modify helper to import utils
|
||||
const helperPyWithImport = `from f.test.utils import get_version
|
||||
|
||||
def helper_func():
|
||||
return get_version()
|
||||
`;
|
||||
await writeFile(`${tempDir}/f/test/helper.py`, helperPyWithImport);
|
||||
|
||||
// Regenerate - main should now have requests
|
||||
const afterAdd = await backend.runCLICommand(
|
||||
["generate-metadata", "-i", "f/test/*", "--yes"],
|
||||
tempDir
|
||||
);
|
||||
expect(afterAdd.code).toBe(0);
|
||||
|
||||
const lockUtils = await readFile(`${tempDir}/f/test/utils.script.lock`, "utf-8").catch(() => "");
|
||||
const lockHelper = await readFile(`${tempDir}/f/test/helper.script.lock`, "utf-8").catch(() => "");
|
||||
const lockMain = await readFile(`${tempDir}/f/test/main.script.lock`, "utf-8").catch(() => "");
|
||||
|
||||
expect(lockUtils).toContain("requests");
|
||||
expect(lockHelper).toContain("requests");
|
||||
expect(lockMain).toContain("requests");
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,322 @@
|
||||
/**
|
||||
* Unit tests for replacePathScriptsWithLocal.
|
||||
*
|
||||
* Tests that PathScript ("script" type) modules are correctly converted
|
||||
* to RawScript ("rawscript" type) using local file content during
|
||||
* flow preview / dev mode.
|
||||
*/
|
||||
|
||||
import { expect, test, describe } from "bun:test";
|
||||
import { replacePathScriptsWithLocal, type LocalScriptInfo } from "../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import type { FlowModule } from "../windmill-utils-internal/src/gen/types.gen.ts";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function makePathScriptModule(
|
||||
id: string,
|
||||
path: string,
|
||||
inputTransforms: Record<string, any> = {},
|
||||
tagOverride?: string,
|
||||
): FlowModule {
|
||||
return {
|
||||
id,
|
||||
value: {
|
||||
type: "script" as const,
|
||||
path,
|
||||
input_transforms: inputTransforms,
|
||||
tag_override: tagOverride,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeRawscriptModule(
|
||||
id: string,
|
||||
content: string,
|
||||
language: "bun" | "python3" | "deno" = "bun",
|
||||
): FlowModule {
|
||||
return {
|
||||
id,
|
||||
value: {
|
||||
type: "rawscript" as const,
|
||||
content,
|
||||
language,
|
||||
input_transforms: {},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const noopLogger = {
|
||||
info: () => {},
|
||||
error: () => {},
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Basic conversion tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("replacePathScriptsWithLocal", () => {
|
||||
test("converts PathScript to RawScript when local file exists", async () => {
|
||||
const module = makePathScriptModule("a", "f/scripts/my_script", {
|
||||
x: { type: "static", value: 42 },
|
||||
});
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/my_script") {
|
||||
return {
|
||||
content: 'export function main() { return "hello"; }',
|
||||
language: "bun",
|
||||
lock: "some-lock",
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
|
||||
expect(module.value.type).toBe("rawscript");
|
||||
expect((module.value as any).content).toBe('export function main() { return "hello"; }');
|
||||
expect((module.value as any).language).toBe("bun");
|
||||
expect((module.value as any).lock).toBe("some-lock");
|
||||
expect((module.value as any).path).toBe("f/scripts/my_script");
|
||||
expect((module.value as any).input_transforms).toEqual({ x: { type: "static", value: 42 } });
|
||||
});
|
||||
|
||||
test("preserves tag_override as tag", async () => {
|
||||
const module = makePathScriptModule("a", "f/scripts/tagged", {}, "my-worker");
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/tagged") {
|
||||
return { content: "code", language: "python3", tag: "script-worker" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
|
||||
expect(module.value.type).toBe("rawscript");
|
||||
expect((module.value as any).tag).toBe("my-worker");
|
||||
});
|
||||
|
||||
test("uses local script tag when no tag_override is set", async () => {
|
||||
const module = makePathScriptModule("a", "f/scripts/tagged");
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/tagged") {
|
||||
return { content: "code", language: "python3", tag: "script-worker" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
|
||||
expect(module.value.type).toBe("rawscript");
|
||||
expect((module.value as any).tag).toBe("script-worker");
|
||||
});
|
||||
|
||||
test("leaves PathScript untouched when local file not found", async () => {
|
||||
const module = makePathScriptModule("a", "f/scripts/remote_only");
|
||||
|
||||
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => undefined;
|
||||
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
|
||||
expect(module.value.type).toBe("script");
|
||||
expect((module.value as any).path).toBe("f/scripts/remote_only");
|
||||
});
|
||||
|
||||
test("does not affect rawscript modules", async () => {
|
||||
const module = makeRawscriptModule("a", "existing code", "bun");
|
||||
|
||||
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => {
|
||||
throw new Error("should not be called for rawscript");
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
|
||||
expect(module.value.type).toBe("rawscript");
|
||||
expect((module.value as any).content).toBe("existing code");
|
||||
});
|
||||
|
||||
test("handles mixed PathScript and RawScript modules", async () => {
|
||||
const pathModule = makePathScriptModule("a", "f/scripts/local_script");
|
||||
const rawModule = makeRawscriptModule("b", "inline code", "bun");
|
||||
const remoteModule = makePathScriptModule("c", "f/scripts/remote_only");
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/local_script") {
|
||||
return { content: "local code", language: "python3" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal(
|
||||
[pathModule, rawModule, remoteModule],
|
||||
scriptReader,
|
||||
noopLogger,
|
||||
);
|
||||
|
||||
// PathScript with local file → converted
|
||||
expect(pathModule.value.type).toBe("rawscript");
|
||||
expect((pathModule.value as any).content).toBe("local code");
|
||||
|
||||
// RawScript → untouched
|
||||
expect(rawModule.value.type).toBe("rawscript");
|
||||
expect((rawModule.value as any).content).toBe("inline code");
|
||||
|
||||
// PathScript without local file → untouched
|
||||
expect(remoteModule.value.type).toBe("script");
|
||||
expect((remoteModule.value as any).path).toBe("f/scripts/remote_only");
|
||||
});
|
||||
|
||||
test("handles module without lock", async () => {
|
||||
const module = makePathScriptModule("a", "f/scripts/no_lock");
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/no_lock") {
|
||||
return { content: "code", language: "go" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
|
||||
expect(module.value.type).toBe("rawscript");
|
||||
expect((module.value as any).lock).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Nested structure tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("replacePathScriptsWithLocal nested structures", () => {
|
||||
test("processes modules inside forloopflow", async () => {
|
||||
const innerModule = makePathScriptModule("inner", "f/scripts/loop_script");
|
||||
const loopModule: FlowModule = {
|
||||
id: "loop",
|
||||
value: {
|
||||
type: "forloopflow" as const,
|
||||
iterator: { type: "static" as const, value: "" },
|
||||
modules: [innerModule],
|
||||
skip_failures: false,
|
||||
},
|
||||
};
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/loop_script") {
|
||||
return { content: "loop code", language: "bun" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([loopModule], scriptReader, noopLogger);
|
||||
|
||||
expect(innerModule.value.type).toBe("rawscript");
|
||||
expect((innerModule.value as any).content).toBe("loop code");
|
||||
});
|
||||
|
||||
test("processes modules inside whileloopflow", async () => {
|
||||
const innerModule = makePathScriptModule("inner", "f/scripts/while_script");
|
||||
const whileModule: FlowModule = {
|
||||
id: "while",
|
||||
value: {
|
||||
type: "whileloopflow" as const,
|
||||
modules: [innerModule],
|
||||
skip_failures: false,
|
||||
},
|
||||
};
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/while_script") {
|
||||
return { content: "while code", language: "bun" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([whileModule], scriptReader, noopLogger);
|
||||
|
||||
expect(innerModule.value.type).toBe("rawscript");
|
||||
expect((innerModule.value as any).content).toBe("while code");
|
||||
});
|
||||
|
||||
test("processes modules inside branchall", async () => {
|
||||
const branch1Module = makePathScriptModule("b1", "f/scripts/branch1");
|
||||
const branch2Module = makePathScriptModule("b2", "f/scripts/branch2");
|
||||
const branchAllModule: FlowModule = {
|
||||
id: "branches",
|
||||
value: {
|
||||
type: "branchall" as const,
|
||||
branches: [
|
||||
{ modules: [branch1Module], skip_failure: false },
|
||||
{ modules: [branch2Module], skip_failure: false },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/branch1") {
|
||||
return { content: "branch1 code", language: "bun" };
|
||||
}
|
||||
if (scriptPath === "f/scripts/branch2") {
|
||||
return { content: "branch2 code", language: "python3" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([branchAllModule], scriptReader, noopLogger);
|
||||
|
||||
expect(branch1Module.value.type).toBe("rawscript");
|
||||
expect((branch1Module.value as any).content).toBe("branch1 code");
|
||||
expect(branch2Module.value.type).toBe("rawscript");
|
||||
expect((branch2Module.value as any).content).toBe("branch2 code");
|
||||
expect((branch2Module.value as any).language).toBe("python3");
|
||||
});
|
||||
|
||||
test("processes modules inside branchone (branches + default)", async () => {
|
||||
const branchModule = makePathScriptModule("b1", "f/scripts/branch");
|
||||
const defaultModule = makePathScriptModule("d1", "f/scripts/default");
|
||||
const branchOneModule: FlowModule = {
|
||||
id: "branchone",
|
||||
value: {
|
||||
type: "branchone" as const,
|
||||
branches: [
|
||||
{ modules: [branchModule], expr: "true" },
|
||||
],
|
||||
default: [defaultModule],
|
||||
},
|
||||
};
|
||||
|
||||
const scriptReader = async (scriptPath: string): Promise<LocalScriptInfo | undefined> => {
|
||||
if (scriptPath === "f/scripts/branch") {
|
||||
return { content: "branch code", language: "bun" };
|
||||
}
|
||||
if (scriptPath === "f/scripts/default") {
|
||||
return { content: "default code", language: "bash" };
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
await replacePathScriptsWithLocal([branchOneModule], scriptReader, noopLogger);
|
||||
|
||||
expect(branchModule.value.type).toBe("rawscript");
|
||||
expect((branchModule.value as any).content).toBe("branch code");
|
||||
expect(defaultModule.value.type).toBe("rawscript");
|
||||
expect((defaultModule.value as any).content).toBe("default code");
|
||||
expect((defaultModule.value as any).language).toBe("bash");
|
||||
});
|
||||
|
||||
test("handles empty modules array", async () => {
|
||||
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => undefined;
|
||||
// Should not throw
|
||||
await replacePathScriptsWithLocal([], scriptReader, noopLogger);
|
||||
});
|
||||
|
||||
test("handles module with undefined value gracefully", async () => {
|
||||
const module: FlowModule = { id: "x", value: undefined as any };
|
||||
const scriptReader = async (): Promise<LocalScriptInfo | undefined> => undefined;
|
||||
// Should not throw
|
||||
await replacePathScriptsWithLocal([module], scriptReader, noopLogger);
|
||||
});
|
||||
});
|
||||
@@ -15,6 +15,7 @@ import {
|
||||
isAppPath,
|
||||
isRawAppPath,
|
||||
isFolderResourcePath,
|
||||
isFolderResourcePathAnyFormat,
|
||||
detectFolderResourceType,
|
||||
isRawAppBackendPath,
|
||||
isAppInlineScriptPath,
|
||||
@@ -32,6 +33,8 @@ import {
|
||||
isRawAppFolderMetadataFile,
|
||||
getDeleteSuffix,
|
||||
transformJsonPathToDir,
|
||||
isModuleEntryPoint,
|
||||
getScriptBasePathFromModulePath,
|
||||
} from "../src/utils/resource_folders.ts";
|
||||
import { removeWorkerPrefix } from "../src/commands/worker-groups/worker-groups.ts";
|
||||
|
||||
@@ -212,6 +215,39 @@ describe("isFolderResourcePath", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// This is the bug that isFolderResourcePathAnyFormat fixes:
|
||||
// when nonDottedPaths is false (default), isFolderResourcePath misses non-dotted paths
|
||||
// like "f/my_raw__raw_app/backend/a.ts", causing raw app backend scripts to leak
|
||||
// into the standalone script list during generate-metadata.
|
||||
describe("isFolderResourcePathAnyFormat", () => {
|
||||
test("detects non-dotted paths even when global setting is dotted", () => {
|
||||
setNonDottedPaths(false);
|
||||
expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true);
|
||||
expect(isFolderResourcePathAnyFormat("f/my_flow__flow/step.ts")).toBe(true);
|
||||
expect(isFolderResourcePathAnyFormat("f/dashboard__app/inline.ts")).toBe(true);
|
||||
});
|
||||
|
||||
test("detects dotted paths even when global setting is non-dotted", () => {
|
||||
setNonDottedPaths(true);
|
||||
expect(isFolderResourcePathAnyFormat("f/my_raw.raw_app/backend/a.ts")).toBe(true);
|
||||
expect(isFolderResourcePathAnyFormat("f/my_flow.flow/step.ts")).toBe(true);
|
||||
expect(isFolderResourcePathAnyFormat("f/dashboard.app/inline.ts")).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects non-folder-resource paths", () => {
|
||||
expect(isFolderResourcePathAnyFormat("f/my_script.ts")).toBe(false);
|
||||
expect(isFolderResourcePathAnyFormat("f/var.variable.yaml")).toBe(false);
|
||||
});
|
||||
|
||||
test("confirms isFolderResourcePath fails for mismatched format (the bug)", () => {
|
||||
setNonDottedPaths(false);
|
||||
// isFolderResourcePath misses non-dotted paths when setting is dotted
|
||||
expect(isFolderResourcePath("f/my_raw__raw_app/backend/a.ts")).toBe(false);
|
||||
// isFolderResourcePathAnyFormat catches it
|
||||
expect(isFolderResourcePathAnyFormat("f/my_raw__raw_app/backend/a.ts")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectFolderResourceType", () => {
|
||||
test("detects flow type", () => {
|
||||
expect(detectFolderResourceType("f/x.flow/flow.yaml")).toBe("flow");
|
||||
@@ -504,6 +540,70 @@ describe("transformJsonPathToDir", () => {
|
||||
// removeWorkerPrefix (from worker-groups.ts)
|
||||
// =============================================================================
|
||||
|
||||
// =============================================================================
|
||||
// Module Path Functions
|
||||
// =============================================================================
|
||||
|
||||
describe("isModuleEntryPoint", () => {
|
||||
test("detects entry point files in __mod folders", () => {
|
||||
expect(isModuleEntryPoint("f/my_script__mod/script.ts")).toBe(true);
|
||||
expect(isModuleEntryPoint("u/admin/tool__mod/script.py")).toBe(true);
|
||||
expect(isModuleEntryPoint("f/nested/path/script__mod/script.go")).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects non-entry-point files in __mod folders", () => {
|
||||
expect(isModuleEntryPoint("f/my_script__mod/helper.ts")).toBe(false);
|
||||
expect(isModuleEntryPoint("f/my_script__mod/utils/math.py")).toBe(false);
|
||||
expect(isModuleEntryPoint("f/my_script__mod/helper.lock")).toBe(false);
|
||||
});
|
||||
|
||||
test("rejects entry point files in subdirectories of __mod", () => {
|
||||
expect(isModuleEntryPoint("f/my_script__mod/sub/script.ts")).toBe(false);
|
||||
});
|
||||
|
||||
test("rejects paths without __mod", () => {
|
||||
expect(isModuleEntryPoint("f/my_script.ts")).toBe(false);
|
||||
expect(isModuleEntryPoint("f/script.ts")).toBe(false);
|
||||
});
|
||||
|
||||
test("handles windows-style separators", () => {
|
||||
expect(isModuleEntryPoint("f\\my_script__mod\\script.ts")).toBe(true);
|
||||
expect(isModuleEntryPoint("f\\my_script__mod\\helper.ts")).toBe(false);
|
||||
});
|
||||
|
||||
test("matches any extension for script.*", () => {
|
||||
expect(isModuleEntryPoint("f/x__mod/script.yaml")).toBe(true);
|
||||
expect(isModuleEntryPoint("f/x__mod/script.json")).toBe(true);
|
||||
expect(isModuleEntryPoint("f/x__mod/script.lock")).toBe(true);
|
||||
expect(isModuleEntryPoint("f/x__mod/script.sh")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("getScriptBasePathFromModulePath", () => {
|
||||
test("extracts base path from module entry point", () => {
|
||||
expect(getScriptBasePathFromModulePath("f/my_script__mod/script.ts")).toBe("f/my_script");
|
||||
expect(getScriptBasePathFromModulePath("u/admin/tool__mod/script.py")).toBe("u/admin/tool");
|
||||
});
|
||||
|
||||
test("extracts base path from module helper files", () => {
|
||||
expect(getScriptBasePathFromModulePath("f/my_script__mod/helper.ts")).toBe("f/my_script");
|
||||
expect(getScriptBasePathFromModulePath("f/my_script__mod/utils/math.py")).toBe("f/my_script");
|
||||
});
|
||||
|
||||
test("extracts base path from nested script paths", () => {
|
||||
expect(getScriptBasePathFromModulePath("f/deeply/nested/script__mod/helper.ts")).toBe("f/deeply/nested/script");
|
||||
});
|
||||
|
||||
test("returns undefined for non-module paths", () => {
|
||||
expect(getScriptBasePathFromModulePath("f/my_script.ts")).toBeUndefined();
|
||||
expect(getScriptBasePathFromModulePath("f/my_flow.flow/flow.yaml")).toBeUndefined();
|
||||
});
|
||||
|
||||
test("handles windows-style separators", () => {
|
||||
expect(getScriptBasePathFromModulePath("f\\my_script__mod\\helper.ts")).toBe("f/my_script");
|
||||
});
|
||||
});
|
||||
|
||||
describe("removeWorkerPrefix", () => {
|
||||
test("removes worker__ prefix", () => {
|
||||
expect(removeWorkerPrefix("worker__default")).toBe("default");
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
/**
|
||||
* Unit tests for script module utilities.
|
||||
* These tests require no backend — they test standalone logic.
|
||||
*/
|
||||
|
||||
import { expect, test, describe, beforeEach, afterEach } from "bun:test";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import * as os from "node:os";
|
||||
import {
|
||||
getModuleFolderSuffix,
|
||||
isScriptModulePath,
|
||||
buildModuleFolderPath,
|
||||
isModuleEntryPoint,
|
||||
getScriptBasePathFromModulePath,
|
||||
} from "../src/utils/resource_folders.ts";
|
||||
import {
|
||||
writeModulesToDisk,
|
||||
readModulesFromDisk,
|
||||
} from "../src/commands/script/script.ts";
|
||||
import { getTypeStrFromPath } from "../src/types.ts";
|
||||
|
||||
// =============================================================================
|
||||
// Module Path Utilities
|
||||
// =============================================================================
|
||||
|
||||
describe("getModuleFolderSuffix", () => {
|
||||
test("returns __mod", () => {
|
||||
expect(getModuleFolderSuffix()).toBe("__mod");
|
||||
});
|
||||
});
|
||||
|
||||
describe("isScriptModulePath", () => {
|
||||
test("detects module paths", () => {
|
||||
expect(isScriptModulePath("f/my_script__mod/helper.ts")).toBe(true);
|
||||
expect(isScriptModulePath("u/admin/script__mod/utils/math.py")).toBe(true);
|
||||
});
|
||||
|
||||
test("rejects non-module paths", () => {
|
||||
expect(isScriptModulePath("f/my_script.ts")).toBe(false);
|
||||
expect(isScriptModulePath("f/my_script__mod")).toBe(false); // no trailing /
|
||||
expect(isScriptModulePath("f/my_flow__flow/flow.yaml")).toBe(false);
|
||||
});
|
||||
|
||||
test("handles windows-style separators", () => {
|
||||
expect(isScriptModulePath("f\\my_script__mod\\helper.ts")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildModuleFolderPath", () => {
|
||||
test("appends __mod suffix", () => {
|
||||
expect(buildModuleFolderPath("f/my_script")).toBe("f/my_script__mod");
|
||||
expect(buildModuleFolderPath("u/admin/tool")).toBe("u/admin/tool__mod");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// writeModulesToDisk / readModulesFromDisk round-trip
|
||||
// =============================================================================
|
||||
|
||||
describe("module read/write round-trip", () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-module-test-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("writes and reads back a simple module", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
const modules = {
|
||||
"helper.ts": {
|
||||
content: 'export function greet() { return "hi"; }\n',
|
||||
language: "bun" as const,
|
||||
},
|
||||
};
|
||||
|
||||
await writeModulesToDisk(moduleFolderPath, modules, "bun");
|
||||
|
||||
// Verify file exists on disk
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "helper.ts"))).toBe(true);
|
||||
|
||||
// Read back
|
||||
const result = await readModulesFromDisk(moduleFolderPath, "bun", false);
|
||||
expect(result).toBeDefined();
|
||||
expect(Object.keys(result!)).toEqual(["helper.ts"]);
|
||||
expect(result!["helper.ts"].content).toBe('export function greet() { return "hi"; }\n');
|
||||
expect(result!["helper.ts"].language).toBe("bun");
|
||||
});
|
||||
|
||||
test("writes and reads back module with lock file", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
const modules = {
|
||||
"helper.py": {
|
||||
content: "def greet():\n return 'hi'\n",
|
||||
language: "python3" as const,
|
||||
lock: "requests==2.31.0\n",
|
||||
},
|
||||
};
|
||||
|
||||
await writeModulesToDisk(moduleFolderPath, modules, undefined);
|
||||
|
||||
// Verify lock file exists
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "helper.lock"))).toBe(true);
|
||||
|
||||
// Read back
|
||||
const result = await readModulesFromDisk(moduleFolderPath, undefined, false);
|
||||
expect(result).toBeDefined();
|
||||
expect(result!["helper.py"].lock).toBe("requests==2.31.0\n");
|
||||
});
|
||||
|
||||
test("writes and reads back nested module paths", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
const modules = {
|
||||
"utils/math.py": {
|
||||
content: "def add(a, b):\n return a + b\n",
|
||||
language: "python3" as const,
|
||||
},
|
||||
"utils/__init__.py": {
|
||||
content: "",
|
||||
language: "python3" as const,
|
||||
},
|
||||
};
|
||||
|
||||
await writeModulesToDisk(moduleFolderPath, modules, undefined);
|
||||
|
||||
// Verify nested structure
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "utils", "math.py"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "utils", "__init__.py"))).toBe(true);
|
||||
|
||||
// Read back
|
||||
const result = await readModulesFromDisk(moduleFolderPath, undefined, false);
|
||||
expect(result).toBeDefined();
|
||||
expect(Object.keys(result!).sort()).toEqual(["utils/__init__.py", "utils/math.py"]);
|
||||
expect(result!["utils/math.py"].content).toBe("def add(a, b):\n return a + b\n");
|
||||
});
|
||||
|
||||
test("returns undefined for non-existent folder", async () => {
|
||||
const result = await readModulesFromDisk(path.join(tempDir, "nonexistent__mod"), undefined, false);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
test("returns undefined for empty folder", async () => {
|
||||
const emptyFolder = path.join(tempDir, "empty__mod");
|
||||
fs.mkdirSync(emptyFolder);
|
||||
const result = await readModulesFromDisk(emptyFolder, undefined, false);
|
||||
expect(result).toBeUndefined();
|
||||
});
|
||||
|
||||
test("multiple modules of different languages", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
const modules = {
|
||||
"helper.ts": {
|
||||
content: "export const x = 1;\n",
|
||||
language: "bun" as const,
|
||||
},
|
||||
"other.ts": {
|
||||
content: "export const y = 2;\n",
|
||||
language: "bun" as const,
|
||||
lock: "some-lock-content\n",
|
||||
},
|
||||
};
|
||||
|
||||
await writeModulesToDisk(moduleFolderPath, modules, "bun");
|
||||
const result = await readModulesFromDisk(moduleFolderPath, "bun", false);
|
||||
|
||||
expect(result).toBeDefined();
|
||||
expect(Object.keys(result!).sort()).toEqual(["helper.ts", "other.ts"]);
|
||||
expect(result!["other.ts"].lock).toBe("some-lock-content\n");
|
||||
expect(result!["helper.ts"].lock).toBeUndefined();
|
||||
});
|
||||
|
||||
test("folder layout skips entry point files", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
fs.mkdirSync(moduleFolderPath, { recursive: true });
|
||||
|
||||
// Simulate folder layout: script.ts is the entry point, helper.ts is a module
|
||||
fs.writeFileSync(path.join(moduleFolderPath, "script.ts"), "export function main() {}\n");
|
||||
fs.writeFileSync(path.join(moduleFolderPath, "script.yaml"), "description: test\n");
|
||||
fs.writeFileSync(path.join(moduleFolderPath, "script.lock"), "");
|
||||
fs.writeFileSync(path.join(moduleFolderPath, "helper.ts"), "export const x = 1;\n");
|
||||
|
||||
// With folderLayout=true, entry point files should be skipped
|
||||
const result = await readModulesFromDisk(moduleFolderPath, "bun", true);
|
||||
expect(result).toBeDefined();
|
||||
expect(Object.keys(result!)).toEqual(["helper.ts"]);
|
||||
|
||||
// With folderLayout=false, all files are included
|
||||
const resultFlat = await readModulesFromDisk(moduleFolderPath, "bun", false);
|
||||
expect(resultFlat).toBeDefined();
|
||||
expect(Object.keys(resultFlat!).sort()).toContain("script.ts");
|
||||
expect(Object.keys(resultFlat!).sort()).toContain("helper.ts");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// getTypeStrFromPath with module paths
|
||||
// =============================================================================
|
||||
|
||||
describe("getTypeStrFromPath with module paths", () => {
|
||||
test("recognizes module content files as scripts", () => {
|
||||
expect(getTypeStrFromPath("f/my_script__mod/helper.ts")).toBe("script");
|
||||
expect(getTypeStrFromPath("u/admin/tool__mod/utils/math.py")).toBe("script");
|
||||
expect(getTypeStrFromPath("f/x__mod/helper.go")).toBe("script");
|
||||
});
|
||||
|
||||
test("recognizes module lock files as scripts", () => {
|
||||
// Lock files inside __mod/ should NOT throw — they are valid module files
|
||||
expect(getTypeStrFromPath("f/my_script__mod/helper.lock")).toBe("script");
|
||||
expect(getTypeStrFromPath("u/admin/tool__mod/utils/math.lock")).toBe("script");
|
||||
});
|
||||
|
||||
test("recognizes module entry points as scripts", () => {
|
||||
expect(getTypeStrFromPath("f/my_script__mod/script.ts")).toBe("script");
|
||||
expect(getTypeStrFromPath("f/my_script__mod/script.py")).toBe("script");
|
||||
});
|
||||
|
||||
test("recognizes module metadata files as scripts", () => {
|
||||
// .yaml/.json files inside __mod/ should be recognized
|
||||
expect(getTypeStrFromPath("f/my_script__mod/script.yaml")).toBe("script");
|
||||
expect(getTypeStrFromPath("f/my_script__mod/script.json")).toBe("script");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Module read/write with multiple lock files
|
||||
// =============================================================================
|
||||
|
||||
describe("module lock file handling", () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "wm-module-lock-test-"));
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("writes and reads multiple modules each with their own lock", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
const modules = {
|
||||
"api_client.py": {
|
||||
content: "import requests\ndef fetch(): pass\n",
|
||||
language: "python3" as const,
|
||||
lock: "requests==2.31.0\nurllib3==2.0.4\n",
|
||||
},
|
||||
"data_processor.py": {
|
||||
content: "import pandas\ndef process(): pass\n",
|
||||
language: "python3" as const,
|
||||
lock: "pandas==2.1.0\nnumpy==1.25.0\n",
|
||||
},
|
||||
"utils.py": {
|
||||
content: "def helper(): pass\n",
|
||||
language: "python3" as const,
|
||||
// No lock — no external deps
|
||||
},
|
||||
};
|
||||
|
||||
await writeModulesToDisk(moduleFolderPath, modules, undefined);
|
||||
|
||||
// Verify lock files exist only for modules with locks
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "api_client.lock"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "data_processor.lock"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "utils.lock"))).toBe(false);
|
||||
|
||||
// Read back and verify
|
||||
const result = await readModulesFromDisk(moduleFolderPath, undefined, false);
|
||||
expect(result).toBeDefined();
|
||||
expect(Object.keys(result!).sort()).toEqual(["api_client.py", "data_processor.py", "utils.py"]);
|
||||
expect(result!["api_client.py"].lock).toBe("requests==2.31.0\nurllib3==2.0.4\n");
|
||||
expect(result!["data_processor.py"].lock).toBe("pandas==2.1.0\nnumpy==1.25.0\n");
|
||||
expect(result!["utils.py"].lock).toBeUndefined();
|
||||
});
|
||||
|
||||
test("nested modules with lock files", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
const modules = {
|
||||
"services/api.ts": {
|
||||
content: "export function callApi() {}\n",
|
||||
language: "bun" as const,
|
||||
lock: "axios@1.5.0\n",
|
||||
},
|
||||
"services/db.ts": {
|
||||
content: "export function query() {}\n",
|
||||
language: "bun" as const,
|
||||
lock: "pg@8.11.0\n",
|
||||
},
|
||||
};
|
||||
|
||||
await writeModulesToDisk(moduleFolderPath, modules, "bun");
|
||||
|
||||
// Verify nested lock files
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "services", "api.lock"))).toBe(true);
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "services", "db.lock"))).toBe(true);
|
||||
|
||||
const result = await readModulesFromDisk(moduleFolderPath, "bun", false);
|
||||
expect(result).toBeDefined();
|
||||
expect(result!["services/api.ts"].lock).toBe("axios@1.5.0\n");
|
||||
expect(result!["services/db.ts"].lock).toBe("pg@8.11.0\n");
|
||||
});
|
||||
|
||||
test("overwrites existing module folder on re-write", async () => {
|
||||
const moduleFolderPath = path.join(tempDir, "my_script__mod");
|
||||
|
||||
// First write
|
||||
await writeModulesToDisk(moduleFolderPath, {
|
||||
"old.ts": { content: "old content\n", language: "bun" as const },
|
||||
}, "bun");
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "old.ts"))).toBe(true);
|
||||
|
||||
// Second write with different module
|
||||
await writeModulesToDisk(moduleFolderPath, {
|
||||
"new.ts": { content: "new content\n", language: "bun" as const },
|
||||
}, "bun");
|
||||
expect(fs.existsSync(path.join(moduleFolderPath, "new.ts"))).toBe(true);
|
||||
// old.ts should still exist (writeModulesToDisk doesn't clean up)
|
||||
// This is intentional — cleanup happens at the sync level
|
||||
});
|
||||
});
|
||||
@@ -409,6 +409,63 @@ describe("script bootstrap command", () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("accepts 'python' as alias for python3", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspaceProfile(backend);
|
||||
|
||||
await writeFile(
|
||||
join(tempDir, "wmill.yaml"),
|
||||
`defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
await mkdir(join(tempDir, "f", "test"), { recursive: true });
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["script", "bootstrap", "f/test/py_alias_script", "python"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
|
||||
const codeStat = await stat(join(tempDir, "f/test/py_alias_script.py"));
|
||||
expect(codeStat.isFile()).toBe(true);
|
||||
|
||||
const metaStat = await stat(
|
||||
join(tempDir, "f/test/py_alias_script.script.yaml")
|
||||
);
|
||||
expect(metaStat.isFile()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test("creates parent directories automatically", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspaceProfile(backend);
|
||||
|
||||
await writeFile(
|
||||
join(tempDir, "wmill.yaml"),
|
||||
`defaultTs: bun\nincludes:\n - "**"\nexcludes: []\n`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Do NOT pre-create f/test — bootstrap should create it
|
||||
const result = await backend.runCLICommand(
|
||||
["script", "bootstrap", "f/test/auto_dir_script", "bun"],
|
||||
tempDir
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
|
||||
const codeStat = await stat(join(tempDir, "f/test/auto_dir_script.ts"));
|
||||
expect(codeStat.isFile()).toBe(true);
|
||||
|
||||
const metaStat = await stat(
|
||||
join(tempDir, "f/test/auto_dir_script.script.yaml")
|
||||
);
|
||||
expect(metaStat.isFile()).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
test("creates Go script files", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspaceProfile(backend);
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
*
|
||||
* Tests the sync pull and push functionality with a simulated filesystem
|
||||
* containing every kind of Windmill resource type.
|
||||
*
|
||||
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
|
||||
* @see test_fixtures.ts - Shared local fixtures (prefer using this module for new tests)
|
||||
* @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource,
|
||||
* createAppWithInlineScript, createFlowWithInlineScript, etc.)
|
||||
*
|
||||
* This file contains: Local fixtures (should migrate to test_fixtures.ts) + createRemoteScript
|
||||
* If you add new helpers, update cross-links in the files above.
|
||||
*/
|
||||
|
||||
import { expect, test, describe } from "bun:test";
|
||||
@@ -32,15 +40,19 @@ import {
|
||||
isAppInlineScriptPath,
|
||||
isFlowInlineScriptPath,
|
||||
isRawAppBackendPath,
|
||||
getModuleFolderSuffix,
|
||||
} from "../src/utils/resource_folders.ts";
|
||||
import { newPathAssigner } from "../windmill-utils-internal/src/path-utils/path-assigner.ts";
|
||||
|
||||
// =============================================================================
|
||||
// Test Fixtures - Every Type of Windmill Resource
|
||||
// See file header for cross-links to related helpers.
|
||||
// Consider migrating these to test_fixtures.ts for reuse across tests.
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates a mock script file structure
|
||||
* Creates a mock script file structure.
|
||||
* See file header for cross-links to related helpers.
|
||||
*/
|
||||
function createScriptFixture(
|
||||
name: string,
|
||||
@@ -89,7 +101,8 @@ kind: script
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a mock flow file structure
|
||||
* Creates a mock flow file structure.
|
||||
* See file header for cross-links to related helpers.
|
||||
*/
|
||||
function createFlowFixture(name: string): Record<string, { path: string; content: string }> {
|
||||
const flowSuffix = getFolderSuffix("flow");
|
||||
@@ -123,7 +136,8 @@ schema:
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a mock app file structure
|
||||
* Creates a mock app file structure.
|
||||
* See file header for cross-links to related helpers.
|
||||
*/
|
||||
function createAppFixture(name: string): Record<string, { path: string; content: string }> {
|
||||
const appSuffix = getFolderSuffix("app");
|
||||
@@ -151,7 +165,8 @@ policy:
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a mock raw_app file structure
|
||||
* Creates a mock raw_app file structure.
|
||||
* See file header for cross-links to related helpers.
|
||||
*/
|
||||
function createRawAppFixture(name: string): Record<string, { path: string; content: string }> {
|
||||
const rawAppSuffix = getFolderSuffix("raw_app");
|
||||
@@ -1755,6 +1770,66 @@ excludes: []
|
||||
});
|
||||
});
|
||||
|
||||
test("Integration: Sync pull with nonDottedPaths uses non-dotted inline script filenames", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
// Push a flow using default dotted paths
|
||||
await writeFile(
|
||||
`${tempDir}/wmill.yaml`,
|
||||
`defaultTs: bun
|
||||
includes:
|
||||
- "**"
|
||||
excludes: []
|
||||
`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const uniqueId = Date.now();
|
||||
const flowName = `f/test/nondot_pull_inline_${uniqueId}`;
|
||||
const flowFixture = createFlowFixture(flowName);
|
||||
await mkdir(`${tempDir}/f/test/nondot_pull_inline_${uniqueId}${getFolderSuffix("flow")}`, { recursive: true });
|
||||
for (const file of Object.values(flowFixture)) {
|
||||
await writeFile(`${tempDir}/${file.path}`, file.content, "utf-8");
|
||||
}
|
||||
|
||||
const pushResult = await backend.runCLICommand(
|
||||
["sync", "push", "--yes", "--includes", `f/test/nondot_pull_inline_${uniqueId}*/**`],
|
||||
tempDir,
|
||||
);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
|
||||
// Pull into a fresh directory with nonDottedPaths enabled
|
||||
const tempDir2 = await mkdtemp(join(tmpdir(), "wmill_nondot_inline_"));
|
||||
try {
|
||||
await writeFile(
|
||||
`${tempDir2}/wmill.yaml`,
|
||||
`defaultTs: bun
|
||||
nonDottedPaths: true
|
||||
includes:
|
||||
- "**"
|
||||
excludes: []
|
||||
`,
|
||||
"utf-8",
|
||||
);
|
||||
|
||||
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir2);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Verify pulled files use non-dotted inline script naming
|
||||
const files = await listFilesRecursive(tempDir2);
|
||||
const flowFiles = files.filter((f) => f.includes(`nondot_pull_inline_${uniqueId}`));
|
||||
|
||||
expect(flowFiles.length > 0).toBeTruthy();
|
||||
// Should use __flow folder, not .flow
|
||||
expect(flowFiles.some((f) => f.includes("__flow/"))).toBeTruthy();
|
||||
// No files should have .inline_script. in their name
|
||||
const dottedInlineFiles = flowFiles.filter((f) => f.includes(".inline_script."));
|
||||
expect(dottedInlineFiles.length).toEqual(0);
|
||||
} finally {
|
||||
await cleanupTempDir(tempDir2);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// ws_error_handler_muted Persistence Tests
|
||||
// =============================================================================
|
||||
@@ -1920,7 +1995,7 @@ excludes: []
|
||||
|
||||
import type { TestBackend } from "./test_backend.ts";
|
||||
|
||||
/** Create a script on the remote via API */
|
||||
/** Create a script on the remote via API. See file header for cross-links. */
|
||||
async function createRemoteScript(
|
||||
backend: TestBackend,
|
||||
scriptPath: string,
|
||||
@@ -2328,3 +2403,315 @@ describe("http trigger sync", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Script Module Sync Tests
|
||||
// =============================================================================
|
||||
|
||||
describe("script module sync", () => {
|
||||
test.skipIf(process.platform === "win32")("push script with modules via API and pull back", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const uniqueId = Date.now();
|
||||
const scriptPath = `f/test/module_script_${uniqueId}`;
|
||||
const modSuffix = getModuleFolderSuffix();
|
||||
|
||||
// Create a script with modules via API
|
||||
const resp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/scripts/create`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path: scriptPath,
|
||||
content: 'import { greet } from "./helper";\nexport async function main() { return greet(); }',
|
||||
language: "bun",
|
||||
summary: "Script with modules",
|
||||
description: "Test script with module files",
|
||||
schema: {
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
modules: {
|
||||
"helper.ts": {
|
||||
content: 'export function greet() { return "hello from module"; }\n',
|
||||
language: "bun",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}
|
||||
);
|
||||
expect(resp.status).toBeLessThan(300);
|
||||
await resp.text();
|
||||
|
||||
await writeWmillYaml(tempDir);
|
||||
|
||||
// Pull the script
|
||||
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Verify module files exist on disk
|
||||
const files = await listFilesRecursive(tempDir);
|
||||
const moduleFile = files.find(
|
||||
(f) => f.includes(`module_script_${uniqueId}${modSuffix}`) && f.endsWith("helper.ts")
|
||||
);
|
||||
expect(moduleFile).toBeDefined();
|
||||
|
||||
// Scripts with modules use folder layout: main content is at __mod/script.ts
|
||||
const mainFile = files.find(
|
||||
(f) => f.includes(`module_script_${uniqueId}${modSuffix}`) && f.endsWith("script.ts")
|
||||
);
|
||||
expect(mainFile).toBeDefined();
|
||||
|
||||
// Verify module content
|
||||
if (moduleFile) {
|
||||
const content = await readFile(`${tempDir}/${moduleFile}`, "utf-8");
|
||||
expect(content).toContain("greet");
|
||||
expect(content).toContain("hello from module");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("push script with module lock files and pull back", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const uniqueId = Date.now();
|
||||
const scriptPath = `f/test/locked_module_${uniqueId}`;
|
||||
const modSuffix = getModuleFolderSuffix();
|
||||
|
||||
// Create a script with a module that has a lock
|
||||
const resp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/scripts/create`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path: scriptPath,
|
||||
content: 'export async function main() { return "main"; }',
|
||||
language: "bun",
|
||||
summary: "Script with locked module",
|
||||
schema: {
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
modules: {
|
||||
"dep.ts": {
|
||||
content: 'import lodash from "lodash";\nexport const x = lodash.identity(1);\n',
|
||||
language: "bun",
|
||||
lock: "lodash@4.17.21\n",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}
|
||||
);
|
||||
expect(resp.status).toBeLessThan(300);
|
||||
await resp.text();
|
||||
|
||||
await writeWmillYaml(tempDir);
|
||||
|
||||
// Pull
|
||||
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Verify lock file exists on disk
|
||||
const files = await listFilesRecursive(tempDir);
|
||||
const lockFile = files.find(
|
||||
(f) => f.includes(`locked_module_${uniqueId}${modSuffix}`) && f.endsWith("dep.lock")
|
||||
);
|
||||
expect(lockFile).toBeDefined();
|
||||
|
||||
// Verify lock content
|
||||
if (lockFile) {
|
||||
const lockContent = await readFile(`${tempDir}/${lockFile}`, "utf-8");
|
||||
expect(lockContent).toContain("lodash");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("local script with modules pushes and round-trips", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const uniqueId = Date.now();
|
||||
const modSuffix = getModuleFolderSuffix();
|
||||
|
||||
await writeWmillYaml(tempDir);
|
||||
|
||||
// Create local script with modules using folder layout:
|
||||
// f/test/modular_<id>__mod/script.ts (entry point)
|
||||
// f/test/modular_<id>__mod/script.yaml (metadata)
|
||||
// f/test/modular_<id>__mod/helper.ts (module file)
|
||||
const scriptDir = `${tempDir}/f/test`;
|
||||
const scriptName = `modular_${uniqueId}`;
|
||||
const modDir = `${scriptDir}/${scriptName}${modSuffix}`;
|
||||
await mkdir(modDir, { recursive: true });
|
||||
|
||||
// Entry point (main script content)
|
||||
await writeFile(
|
||||
`${modDir}/script.ts`,
|
||||
'import { helper } from "./helper";\nexport async function main() { return helper(); }',
|
||||
"utf-8"
|
||||
);
|
||||
// Script metadata
|
||||
await writeFile(
|
||||
`${modDir}/script.yaml`,
|
||||
`summary: "Script with modules"
|
||||
description: "Test"
|
||||
schema:
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema"
|
||||
type: object
|
||||
properties: {}
|
||||
required: []
|
||||
is_template: false
|
||||
lock: ""
|
||||
kind: script
|
||||
`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Module file
|
||||
await writeFile(
|
||||
`${modDir}/helper.ts`,
|
||||
'export function helper() { return "from module"; }\n',
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Push
|
||||
const pushResult = await backend.runCLICommand(["sync", "push", "--yes"], tempDir);
|
||||
expect(pushResult.code).toEqual(0);
|
||||
|
||||
// Pull into a fresh directory to verify round-trip
|
||||
const pullDir = await mkdtemp(join(tmpdir(), "windmill_module_pull_"));
|
||||
try {
|
||||
await writeWmillYaml(pullDir);
|
||||
|
||||
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], pullDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Verify module file came back
|
||||
const files = await listFilesRecursive(pullDir);
|
||||
const pulledModule = files.find(
|
||||
(f) => f.includes(`${scriptName}${modSuffix}`) && f.endsWith("helper.ts")
|
||||
);
|
||||
expect(pulledModule).toBeDefined();
|
||||
|
||||
if (pulledModule) {
|
||||
const content = await readFile(`${pullDir}/${pulledModule}`, "utf-8");
|
||||
expect(content).toContain("from module");
|
||||
}
|
||||
} finally {
|
||||
await rm(pullDir, { recursive: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test("script with nested module paths pushes and pulls correctly", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const uniqueId = Date.now();
|
||||
const scriptPath = `f/test/nested_mod_${uniqueId}`;
|
||||
const modSuffix = getModuleFolderSuffix();
|
||||
|
||||
// Create script with nested modules via API
|
||||
const resp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/scripts/create`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path: scriptPath,
|
||||
content: 'export async function main() { return "main"; }',
|
||||
language: "bun",
|
||||
summary: "Script with nested modules",
|
||||
schema: {
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
modules: {
|
||||
"utils/format.ts": {
|
||||
content: 'export function format(s: string) { return s.trim(); }\n',
|
||||
language: "bun",
|
||||
},
|
||||
"utils/validate.ts": {
|
||||
content: 'export function validate(s: string) { return s.length > 0; }\n',
|
||||
language: "bun",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}
|
||||
);
|
||||
expect(resp.status).toBeLessThan(300);
|
||||
await resp.text();
|
||||
|
||||
await writeWmillYaml(tempDir);
|
||||
|
||||
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Verify nested module files exist
|
||||
const files = await listFilesRecursive(tempDir);
|
||||
const formatFile = files.find(
|
||||
(f) => f.includes(`nested_mod_${uniqueId}${modSuffix}`) && f.includes("utils/format.ts")
|
||||
);
|
||||
const validateFile = files.find(
|
||||
(f) => f.includes(`nested_mod_${uniqueId}${modSuffix}`) && f.includes("utils/validate.ts")
|
||||
);
|
||||
expect(formatFile).toBeDefined();
|
||||
expect(validateFile).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
test.skipIf(process.platform === "win32")("pull script with modules does not create stale metadata", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
const uniqueId = Date.now();
|
||||
const scriptPath = `f/test/fresh_mod_${uniqueId}`;
|
||||
|
||||
// Create a script with modules on remote
|
||||
const resp = await backend.apiRequest!(
|
||||
`/api/w/${backend.workspace}/scripts/create`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path: scriptPath,
|
||||
content: 'export async function main() { return "main"; }',
|
||||
language: "bun",
|
||||
summary: "Script with modules for freshness test",
|
||||
schema: {
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
modules: {
|
||||
"helper.ts": {
|
||||
content: 'export function help() { return true; }\n',
|
||||
language: "bun",
|
||||
},
|
||||
},
|
||||
}),
|
||||
}
|
||||
);
|
||||
expect(resp.status).toBeLessThan(300);
|
||||
await resp.text();
|
||||
|
||||
await writeWmillYaml(tempDir);
|
||||
|
||||
// Pull
|
||||
const pullResult = await backend.runCLICommand(["sync", "pull", "--yes"], tempDir);
|
||||
expect(pullResult.code).toEqual(0);
|
||||
|
||||
// Run generate-metadata — should show up-to-date for pulled scripts
|
||||
const metaResult = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir
|
||||
);
|
||||
expect(metaResult.code).toEqual(0);
|
||||
// The script we just pulled should NOT be stale
|
||||
// (it may still show other scripts as stale from seedTestData)
|
||||
const output = metaResult.stdout + metaResult.stderr;
|
||||
expect(output).not.toContain(`fresh_mod_${uniqueId}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,6 +19,14 @@
|
||||
* // ...
|
||||
* });
|
||||
* });
|
||||
*
|
||||
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
|
||||
* @see test_fixtures.ts - Local file fixtures (createLocalScript, createLocalFlow, etc.)
|
||||
* @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based)
|
||||
*
|
||||
* This file contains: API-based creation helpers (createTestApp, createTestResource,
|
||||
* createAppWithInlineScript, createFlowWithInlineScript, etc.)
|
||||
* If you add new helpers, update cross-links in the files above.
|
||||
*/
|
||||
|
||||
import { CargoBackend, CargoBackendConfig } from "./cargo_backend.ts";
|
||||
@@ -57,6 +65,10 @@ export interface TestBackend {
|
||||
listAllApps?(): Promise<any[]>;
|
||||
listAllResources?(): Promise<any[]>;
|
||||
listAllVariables?(): Promise<any[]>;
|
||||
|
||||
// Methods for creating apps and flows with custom inline scripts
|
||||
createAppWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise<void>;
|
||||
createFlowWithInlineScript?(path: string, inlineScriptContent: string, language?: string): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -109,6 +121,7 @@ class CargoBackendAdapter implements TestBackend {
|
||||
return this.backend.apiRequest(path, options);
|
||||
}
|
||||
|
||||
/** Seeds test data via API calls. See file header for cross-links to related helpers. */
|
||||
async seedTestData(): Promise<void> {
|
||||
// Create test folder first
|
||||
await this.createTestFolder("test");
|
||||
@@ -124,6 +137,7 @@ class CargoBackendAdapter implements TestBackend {
|
||||
await this.createTestApp("f/test/test_dashboard");
|
||||
}
|
||||
|
||||
/** See file header for cross-links to related helpers. */
|
||||
private async createTestApp(path: string): Promise<void> {
|
||||
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, {
|
||||
method: "POST",
|
||||
@@ -156,6 +170,7 @@ class CargoBackendAdapter implements TestBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/** See file header for cross-links to related helpers. */
|
||||
private async createTestFolder(name: string): Promise<void> {
|
||||
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/folders/create`, {
|
||||
method: "POST",
|
||||
@@ -172,6 +187,7 @@ class CargoBackendAdapter implements TestBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/** See file header for cross-links to related helpers. */
|
||||
private async createTestGroup(name: string): Promise<void> {
|
||||
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/groups/create`, {
|
||||
method: "POST",
|
||||
@@ -189,6 +205,7 @@ class CargoBackendAdapter implements TestBackend {
|
||||
}
|
||||
|
||||
|
||||
/** See file header for cross-links to related helpers. */
|
||||
private async createTestResource(path: string, description: string): Promise<void> {
|
||||
// First ensure the folder exists
|
||||
const folderPath = path.split("/").slice(0, 2).join("/"); // e.g., "f/test"
|
||||
@@ -228,6 +245,7 @@ class CargoBackendAdapter implements TestBackend {
|
||||
}
|
||||
}
|
||||
|
||||
/** See file header for cross-links to related helpers. */
|
||||
private async createTestVariable(path: string, value: string): Promise<void> {
|
||||
const response = await this.backend.apiRequest(
|
||||
`/api/w/${this.workspace}/variables/create`,
|
||||
@@ -329,6 +347,88 @@ class CargoBackendAdapter implements TestBackend {
|
||||
if (!response.ok) return [];
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async createAppWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise<void> {
|
||||
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/apps/create`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path,
|
||||
value: {
|
||||
type: "app",
|
||||
grid: [
|
||||
{
|
||||
id: "button1",
|
||||
data: {
|
||||
type: "buttoncomponent",
|
||||
componentInput: {
|
||||
type: "runnable",
|
||||
runnable: {
|
||||
type: "runnableByName",
|
||||
inlineScript: {
|
||||
content: inlineScriptContent,
|
||||
language,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
],
|
||||
hiddenInlineScripts: [],
|
||||
css: {},
|
||||
norefreshbar: false,
|
||||
},
|
||||
summary: "Test app with inline script",
|
||||
policy: {
|
||||
on_behalf_of: null,
|
||||
on_behalf_of_email: null,
|
||||
triggerables: {},
|
||||
execution_mode: "viewer",
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`Failed to create app ${path}: ${error}`);
|
||||
}
|
||||
await response.text();
|
||||
}
|
||||
|
||||
async createFlowWithInlineScript(path: string, inlineScriptContent: string, language: string = "bun"): Promise<void> {
|
||||
const response = await this.backend.apiRequest(`/api/w/${this.workspace}/flows/create`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
path,
|
||||
summary: "Test flow with inline script",
|
||||
description: `Flow at ${path}`,
|
||||
value: {
|
||||
modules: [
|
||||
{
|
||||
id: "a",
|
||||
value: {
|
||||
type: "rawscript",
|
||||
content: inlineScriptContent,
|
||||
language,
|
||||
input_transforms: {},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
schema: {
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
type: "object",
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.text();
|
||||
throw new Error(`Failed to create flow ${path}: ${error}`);
|
||||
}
|
||||
await response.text();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -567,6 +667,38 @@ export async function createNonAdminUser(
|
||||
return await loginResp.text();
|
||||
}
|
||||
|
||||
/**
|
||||
* Create workspace dependencies via the API (e.g. a shared package.json for bun scripts).
|
||||
*/
|
||||
export async function createRemoteWorkspaceDeps(
|
||||
backend: TestBackend,
|
||||
language: string,
|
||||
content: string,
|
||||
name?: string,
|
||||
): Promise<void> {
|
||||
if (!backend.apiRequest) {
|
||||
throw new Error("Backend does not support apiRequest");
|
||||
}
|
||||
|
||||
const resp = await backend.apiRequest(
|
||||
`/api/w/${backend.workspace}/workspace_dependencies/create`,
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
workspace_id: backend.workspace,
|
||||
language,
|
||||
content,
|
||||
...(name ? { name } : {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
if (!resp.ok) {
|
||||
throw new Error(`Failed to create workspace deps (${resp.status}): ${await resp.text()}`);
|
||||
}
|
||||
await resp.text();
|
||||
}
|
||||
|
||||
// Re-export for convenience
|
||||
export type { CargoBackendConfig } from "./cargo_backend.ts";
|
||||
export type { ContainerConfig } from "./containerized_backend.ts";
|
||||
|
||||
@@ -0,0 +1,632 @@
|
||||
/**
|
||||
* Test Fixtures
|
||||
*
|
||||
* Shared helpers for creating test data (scripts, flows, apps, raw apps) in tests.
|
||||
*
|
||||
* Two types of helpers:
|
||||
* - Fixture functions: Return data structures with paths and contents (no disk I/O)
|
||||
* - Local creation functions: Create fixtures AND write them to disk
|
||||
*
|
||||
* CROSS-LINKS - Related test helper locations (keep in sync when adding new helpers):
|
||||
* @see test_backend.ts - API-based creation helpers (createTestApp, createTestResource,
|
||||
* createAppWithInlineScript, createFlowWithInlineScript, etc.)
|
||||
* @see sync_pull_push.test.ts - Local fixtures + createRemoteScript (API-based)
|
||||
*
|
||||
* This file contains: Shared local fixtures (createLocalScript, createLocalFlow, etc.)
|
||||
* If you add new helpers, update cross-links in the files above.
|
||||
*
|
||||
* @example
|
||||
* // Using fixtures (data only)
|
||||
* const fixture = createScriptFixture("my_script", "bun");
|
||||
*
|
||||
* // Using local creation (writes to disk)
|
||||
* await createLocalScript(tempDir, "f/test", "my_script", "bun");
|
||||
*
|
||||
* @keywords createLocal, local script, local flow, local app, raw app, fixture, test data
|
||||
*/
|
||||
|
||||
import { writeFile, mkdir } from "node:fs/promises";
|
||||
import {
|
||||
getFolderSuffix,
|
||||
getMetadataFileName,
|
||||
getModuleFolderSuffix,
|
||||
getNonDottedPaths,
|
||||
} from "../src/utils/resource_folders.ts";
|
||||
|
||||
// =============================================================================
|
||||
// Fixture Types
|
||||
// =============================================================================
|
||||
|
||||
export interface FileFixture {
|
||||
path: string;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface ScriptFixture {
|
||||
contentFile: FileFixture;
|
||||
metadataFile: FileFixture;
|
||||
}
|
||||
|
||||
export interface FlowFixture {
|
||||
metadata: FileFixture;
|
||||
inlineScript?: FileFixture;
|
||||
inlineLock?: FileFixture;
|
||||
}
|
||||
|
||||
export interface AppFixture {
|
||||
metadata: FileFixture;
|
||||
}
|
||||
|
||||
export interface RawAppFixture {
|
||||
metadata: FileFixture;
|
||||
indexHtml: FileFixture;
|
||||
indexJs: FileFixture;
|
||||
[key: string]: FileFixture;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Script Fixtures
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates a script fixture (data structure, no disk I/O).
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* Use this when you need fine-grained control over the script structure.
|
||||
* For simple cases, use {@link createLocalScript} instead.
|
||||
*
|
||||
* @param name - Script name (without extension)
|
||||
* @param language - Script language
|
||||
* @param content - Optional custom script content
|
||||
* @returns Script fixture with content and metadata files
|
||||
*
|
||||
* @example
|
||||
* const fixture = createScriptFixture("my_script", "bun");
|
||||
* const fixture = createScriptFixture("custom", "python3", "def main(): return 42");
|
||||
*
|
||||
* @keywords script fixture, create script, local script
|
||||
*/
|
||||
export function createScriptFixture(
|
||||
name: string,
|
||||
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
|
||||
content?: string
|
||||
): ScriptFixture {
|
||||
const extensions: Record<string, string> = {
|
||||
python3: ".py",
|
||||
deno: ".ts",
|
||||
bun: ".ts",
|
||||
bash: ".sh",
|
||||
go: ".go",
|
||||
postgresql: ".sql",
|
||||
};
|
||||
|
||||
const ext = extensions[language];
|
||||
const defaultContent: Record<string, string> = {
|
||||
python3: `def main():\n return "Hello from ${name}"`,
|
||||
deno: `export async function main() {\n return "Hello from ${name}";\n}`,
|
||||
bun: `export async function main() {\n return "Hello from ${name}";\n}`,
|
||||
bash: `#!/bin/bash\necho "Hello from ${name}"`,
|
||||
go: `package inner\n\nfunc main() string {\n return "Hello from ${name}"\n}`,
|
||||
postgresql: `-- ${name}\nSELECT 'Hello from ${name}';`,
|
||||
};
|
||||
|
||||
return {
|
||||
contentFile: {
|
||||
path: `${name}${ext}`,
|
||||
content: content ?? defaultContent[language],
|
||||
},
|
||||
metadataFile: {
|
||||
path: `${name}.script.yaml`,
|
||||
content: `summary: "${name} script"
|
||||
description: "A ${language} script for testing"
|
||||
schema:
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema"
|
||||
type: object
|
||||
properties: {}
|
||||
required: []
|
||||
is_template: false
|
||||
lock: ""
|
||||
kind: script
|
||||
`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Flow Fixtures
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates a flow fixture (data structure, no disk I/O).
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* TODO: Add optional params: language, summary, description
|
||||
*
|
||||
* Use this when you need fine-grained control over the flow structure.
|
||||
* For simple cases, use {@link createLocalFlow} instead.
|
||||
*
|
||||
* @param name - Flow name
|
||||
* @param inlineScriptContent - Optional custom inline script content
|
||||
* @returns Flow fixture with metadata and inline script
|
||||
*
|
||||
* @example
|
||||
* const fixture = createFlowFixture("my_flow");
|
||||
*
|
||||
* @keywords flow fixture, create flow, local flow
|
||||
*/
|
||||
export function createFlowFixture(
|
||||
name: string,
|
||||
inlineScriptContent?: string,
|
||||
language: "bun" | "python3" = "bun",
|
||||
lockContent?: string
|
||||
): FlowFixture {
|
||||
const flowSuffix = getFolderSuffix("flow");
|
||||
const metadataFile = getMetadataFileName("flow", "yaml");
|
||||
|
||||
const defaultContent = language === "python3"
|
||||
? `def main():\n return "Hello from flow ${name}"`
|
||||
: `export async function main() {\n return "Hello from flow ${name}";\n}`;
|
||||
|
||||
const scriptContent = inlineScriptContent ?? defaultContent;
|
||||
|
||||
const langMap: Record<string, string> = { bun: "bun", python3: "python3" };
|
||||
const extMap: Record<string, string> = { bun: "ts", python3: "py" };
|
||||
|
||||
const ext = extMap[language];
|
||||
// With dotted paths (.flow), inline scripts use .inline_script suffix (a.inline_script.ts)
|
||||
// With non-dotted paths (__flow), they don't (a.ts)
|
||||
const inlineSuffix = getNonDottedPaths() ? "" : ".inline_script";
|
||||
const scriptFile = `a${inlineSuffix}.${ext}`;
|
||||
const lockFile = `a${inlineSuffix}.lock`;
|
||||
|
||||
const lockLine = lockContent !== undefined
|
||||
? `\n lock: "!inline ${lockFile}"`
|
||||
: "";
|
||||
|
||||
const result: FlowFixture = {
|
||||
metadata: {
|
||||
path: `${name}${flowSuffix}/${metadataFile}`,
|
||||
content: `summary: "${name} flow"
|
||||
description: "A flow for testing"
|
||||
value:
|
||||
modules:
|
||||
- id: a
|
||||
value:
|
||||
type: rawscript
|
||||
content: "!inline ${scriptFile}"${lockLine}
|
||||
language: ${langMap[language]}
|
||||
input_transforms: {}
|
||||
schema:
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema"
|
||||
type: object
|
||||
properties: {}
|
||||
required: []
|
||||
`,
|
||||
},
|
||||
inlineScript: {
|
||||
path: `${name}${flowSuffix}/${scriptFile}`,
|
||||
content: scriptContent,
|
||||
},
|
||||
};
|
||||
|
||||
if (lockContent !== undefined) {
|
||||
result.inlineLock = {
|
||||
path: `${name}${flowSuffix}/${lockFile}`,
|
||||
content: lockContent,
|
||||
};
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// App Fixtures
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates an app fixture (data structure, no disk I/O).
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* TODO: Add optional params: inlineScriptContent, summary, grid
|
||||
*
|
||||
* Use this when you need fine-grained control over the app structure.
|
||||
* For simple cases, use {@link createLocalApp} instead.
|
||||
*
|
||||
* @param name - App name
|
||||
* @returns App fixture with metadata
|
||||
*
|
||||
* @example
|
||||
* const fixture = createAppFixture("my_app");
|
||||
*
|
||||
* @keywords app fixture, create app, local app
|
||||
*/
|
||||
export function createAppFixture(name: string, inlineScriptContent?: string): AppFixture {
|
||||
const appSuffix = getFolderSuffix("app");
|
||||
const metadataFile = getMetadataFileName("app", "yaml");
|
||||
|
||||
const scriptContent = inlineScriptContent ??
|
||||
`export async function main() {\n return "hello from app";\n}`;
|
||||
const indented = scriptContent.split("\n").join("\n ");
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
path: `${name}${appSuffix}/${metadataFile}`,
|
||||
content: `summary: "${name} app"
|
||||
value:
|
||||
type: app
|
||||
grid:
|
||||
- id: button1
|
||||
data:
|
||||
type: buttoncomponent
|
||||
componentInput:
|
||||
type: runnable
|
||||
runnable:
|
||||
type: runnableByName
|
||||
inlineScript:
|
||||
content: |
|
||||
${indented}
|
||||
language: bun
|
||||
hiddenInlineScripts: []
|
||||
css: {}
|
||||
norefreshbar: false
|
||||
policy:
|
||||
on_behalf_of: null
|
||||
on_behalf_of_email: null
|
||||
triggerables: {}
|
||||
execution_mode: viewer
|
||||
`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Raw App Fixtures
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates a raw app fixture (data structure, no disk I/O).
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* TODO: Add optional params: inlineScriptContent, htmlContent, jsContent
|
||||
*
|
||||
* Raw apps are React/frontend apps with separate inline scripts.
|
||||
* Use this when you need fine-grained control over the raw app structure.
|
||||
* For simple cases, use {@link createLocalRawApp} instead.
|
||||
*
|
||||
* @param name - Raw app name
|
||||
* @returns Raw app fixture with metadata and frontend files
|
||||
*
|
||||
* @example
|
||||
* const fixture = createRawAppFixture("my_raw_app");
|
||||
*
|
||||
* @keywords raw app fixture, create raw app, local raw app, react app
|
||||
*/
|
||||
export function createRawAppFixture(name: string, inlineScriptContent?: string): RawAppFixture {
|
||||
const rawAppSuffix = getFolderSuffix("raw_app");
|
||||
const metadataFile = getMetadataFileName("raw_app", "yaml");
|
||||
|
||||
const scriptContent = inlineScriptContent ??
|
||||
`export async function main(x: string) {\n return x\n}`;
|
||||
|
||||
return {
|
||||
metadata: {
|
||||
path: `${name}${rawAppSuffix}/${metadataFile}`,
|
||||
content: `summary: "${name} raw app"
|
||||
policy:
|
||||
execution_mode: publisher
|
||||
triggerables: {}
|
||||
`,
|
||||
},
|
||||
indexHtml: {
|
||||
path: `${name}${rawAppSuffix}/index.html`,
|
||||
content: `<!DOCTYPE html>
|
||||
<html>
|
||||
<head><title>${name}</title></head>
|
||||
<body><div id="root"></div></body>
|
||||
</html>`,
|
||||
},
|
||||
indexJs: {
|
||||
path: `${name}${rawAppSuffix}/index.tsx`,
|
||||
content: `import React from 'react'
|
||||
import { createRoot } from 'react-dom/client'
|
||||
|
||||
const App = () => <div><h1>${name}</h1></div>
|
||||
|
||||
const root = createRoot(document.getElementById('root')!)
|
||||
root.render(<App/>)
|
||||
`,
|
||||
},
|
||||
packageJson: {
|
||||
path: `${name}${rawAppSuffix}/package.json`,
|
||||
content: `{
|
||||
"dependencies": {
|
||||
"react": "19.0.0",
|
||||
"react-dom": "19.0.0"
|
||||
}
|
||||
}`,
|
||||
},
|
||||
inlineScript: {
|
||||
path: `${name}${rawAppSuffix}/backend/a.ts`,
|
||||
content: scriptContent + "\n",
|
||||
},
|
||||
inlineScriptMeta: {
|
||||
path: `${name}${rawAppSuffix}/backend/a.yaml`,
|
||||
content: `type: inline\n`,
|
||||
},
|
||||
inlineScriptLock: {
|
||||
path: `${name}${rawAppSuffix}/backend/a.lock`,
|
||||
content: ``,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Local Creation Functions (Fixture + Write to Disk)
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates a script on the local filesystem.
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* This is a convenience function that creates a script fixture and writes it to disk.
|
||||
*
|
||||
* @param tempDir - Base directory for the test workspace
|
||||
* @param path - Relative path within the workspace (e.g., "f/test")
|
||||
* @param name - Script name (without extension)
|
||||
* @param language - Script language (default: "bun")
|
||||
* @param content - Optional custom script content
|
||||
*
|
||||
* @example
|
||||
* await createLocalScript(tempDir, "f/test", "my_script");
|
||||
* await createLocalScript(tempDir, "f/test", "custom", "python3", "def main(): return 42");
|
||||
*
|
||||
* @keywords create local script, local script, write script, script on disk
|
||||
*/
|
||||
export async function createLocalScript(
|
||||
tempDir: string,
|
||||
path: string,
|
||||
name: string,
|
||||
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
|
||||
content?: string
|
||||
): Promise<void> {
|
||||
const fixture = createScriptFixture(name, language, content);
|
||||
await mkdir(`${tempDir}/${path}`, { recursive: true });
|
||||
await writeFile(
|
||||
`${tempDir}/${path}/${fixture.contentFile.path}`,
|
||||
fixture.contentFile.content,
|
||||
"utf-8"
|
||||
);
|
||||
await writeFile(
|
||||
`${tempDir}/${path}/${fixture.metadataFile.path}`,
|
||||
fixture.metadataFile.content,
|
||||
"utf-8"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a flow on the local filesystem.
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* This is a convenience function that creates a flow fixture and writes it to disk.
|
||||
*
|
||||
* @param tempDir - Base directory for the test workspace
|
||||
* @param path - Relative path within the workspace (e.g., "f/test")
|
||||
* @param name - Flow name
|
||||
* @param inlineScriptContent - Optional custom inline script content
|
||||
*
|
||||
* @example
|
||||
* await createLocalFlow(tempDir, "f/test", "my_flow");
|
||||
*
|
||||
* @keywords create local flow, local flow, write flow, flow on disk
|
||||
*/
|
||||
export async function createLocalFlow(
|
||||
tempDir: string,
|
||||
path: string,
|
||||
name: string,
|
||||
inlineScriptContent?: string,
|
||||
language: "bun" | "python3" = "bun",
|
||||
lockContent?: string
|
||||
): Promise<void> {
|
||||
const fixture = createFlowFixture(name, inlineScriptContent, language, lockContent);
|
||||
const flowDir = `${tempDir}/${path}/${name}${getFolderSuffix("flow")}`;
|
||||
await mkdir(flowDir, { recursive: true });
|
||||
|
||||
for (const file of Object.values(fixture)) {
|
||||
if (!file) continue;
|
||||
const fullPath = `${tempDir}/${path}/${file.path}`;
|
||||
await writeFile(fullPath, file.content, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an app on the local filesystem.
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* This is a convenience function that creates an app fixture and writes it to disk.
|
||||
*
|
||||
* @param tempDir - Base directory for the test workspace
|
||||
* @param path - Relative path within the workspace (e.g., "f/test")
|
||||
* @param name - App name
|
||||
*
|
||||
* @example
|
||||
* await createLocalApp(tempDir, "f/test", "my_app");
|
||||
*
|
||||
* @keywords create local app, local app, write app, app on disk
|
||||
*/
|
||||
export async function createLocalApp(
|
||||
tempDir: string,
|
||||
path: string,
|
||||
name: string,
|
||||
inlineScriptContent?: string
|
||||
): Promise<void> {
|
||||
const fixture = createAppFixture(name, inlineScriptContent);
|
||||
const appDir = `${tempDir}/${path}/${name}${getFolderSuffix("app")}`;
|
||||
await mkdir(appDir, { recursive: true });
|
||||
|
||||
for (const file of Object.values(fixture)) {
|
||||
if (!file) continue;
|
||||
const fullPath = `${tempDir}/${path}/${file.path}`;
|
||||
await writeFile(fullPath, file.content, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a raw app on the local filesystem.
|
||||
* See file header for cross-links to related helpers.
|
||||
*
|
||||
* Raw apps are React/frontend apps with separate inline scripts.
|
||||
* This is a convenience function that creates a raw app fixture and writes it to disk.
|
||||
*
|
||||
* @param tempDir - Base directory for the test workspace
|
||||
* @param path - Relative path within the workspace (e.g., "f/test")
|
||||
* @param name - Raw app name
|
||||
*
|
||||
* @example
|
||||
* await createLocalRawApp(tempDir, "f/test", "my_raw_app");
|
||||
*
|
||||
* @keywords create local raw app, local raw app, write raw app, raw app on disk, react app
|
||||
*/
|
||||
export async function createLocalRawApp(
|
||||
tempDir: string,
|
||||
path: string,
|
||||
name: string,
|
||||
inlineScriptContent?: string
|
||||
): Promise<void> {
|
||||
const fixture = createRawAppFixture(name, inlineScriptContent);
|
||||
const rawAppSuffix = getFolderSuffix("raw_app");
|
||||
const appDir = `${tempDir}/${path}/${name}${rawAppSuffix}`;
|
||||
await mkdir(`${appDir}/backend`, { recursive: true });
|
||||
|
||||
for (const file of Object.values(fixture)) {
|
||||
const fullPath = `${tempDir}/${path}/${file.path}`;
|
||||
const dir = fullPath.substring(0, fullPath.lastIndexOf("/"));
|
||||
await mkdir(dir, { recursive: true });
|
||||
await writeFile(fullPath, file.content, "utf-8");
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Script with Modules Fixtures
|
||||
// =============================================================================
|
||||
|
||||
export interface ModuleFile {
|
||||
path: string;
|
||||
content: string;
|
||||
lock?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a script with module files on the local filesystem.
|
||||
*
|
||||
* Creates the main script, its metadata, and module files in a __mod/ folder.
|
||||
* Optionally includes lock files for modules.
|
||||
*
|
||||
* @param tempDir - Base directory for the test workspace
|
||||
* @param dir - Relative path within the workspace (e.g., "f/test")
|
||||
* @param name - Script name (without extension)
|
||||
* @param language - Script language (default: "bun")
|
||||
* @param modules - Module files to create
|
||||
*
|
||||
* @example
|
||||
* await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
|
||||
* { path: "helper.ts", content: "export const x = 1;" },
|
||||
* { path: "utils/math.ts", content: "export function add(a, b) { return a + b; }", lock: "lodash@4.0.0\n" },
|
||||
* ]);
|
||||
*/
|
||||
export async function createLocalScriptWithModules(
|
||||
tempDir: string,
|
||||
dir: string,
|
||||
name: string,
|
||||
language: "python3" | "deno" | "bun" | "bash" | "go" | "postgresql" = "bun",
|
||||
modules: ModuleFile[]
|
||||
): Promise<void> {
|
||||
// Create the main script
|
||||
await createLocalScript(tempDir, dir, name, language);
|
||||
|
||||
// Create module files in __mod/ folder
|
||||
const modSuffix = getModuleFolderSuffix();
|
||||
const modDir = `${tempDir}/${dir}/${name}${modSuffix}`;
|
||||
|
||||
for (const mod of modules) {
|
||||
const fullPath = `${modDir}/${mod.path}`;
|
||||
const parentDir = fullPath.substring(0, fullPath.lastIndexOf("/"));
|
||||
await mkdir(parentDir, { recursive: true });
|
||||
await writeFile(fullPath, mod.content, "utf-8");
|
||||
|
||||
// Write lock file if provided
|
||||
if (mod.lock) {
|
||||
const baseName = mod.path.substring(0, mod.path.indexOf("."));
|
||||
const lockPath = `${modDir}/${baseName}.lock`;
|
||||
const lockDir = lockPath.substring(0, lockPath.lastIndexOf("/"));
|
||||
await mkdir(lockDir, { recursive: true });
|
||||
await writeFile(lockPath, mod.lock, "utf-8");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Resource Fixtures (Variables, Resources, Schedules, etc.)
|
||||
// =============================================================================
|
||||
|
||||
/**
|
||||
* Creates a resource fixture.
|
||||
*
|
||||
* @keywords resource fixture, create resource
|
||||
*/
|
||||
export function createResourceFixture(
|
||||
name: string,
|
||||
resourceType: string,
|
||||
value: Record<string, unknown>
|
||||
): FileFixture {
|
||||
return {
|
||||
path: `${name}.resource.yaml`,
|
||||
content: `resource_type: "${resourceType}"
|
||||
value:
|
||||
${Object.entries(value)
|
||||
.map(([k, v]) => ` ${k}: ${JSON.stringify(v)}`)
|
||||
.join("\n")}
|
||||
`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a variable fixture.
|
||||
*
|
||||
* @keywords variable fixture, create variable
|
||||
*/
|
||||
export function createVariableFixture(
|
||||
name: string,
|
||||
value: string,
|
||||
isSecret: boolean = false
|
||||
): FileFixture {
|
||||
return {
|
||||
path: `${name}.variable.yaml`,
|
||||
content: `value: "${value}"
|
||||
is_secret: ${isSecret}
|
||||
description: "Variable ${name} for testing"
|
||||
`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a schedule fixture.
|
||||
*
|
||||
* @keywords schedule fixture, create schedule
|
||||
*/
|
||||
export function createScheduleFixture(
|
||||
name: string,
|
||||
scriptPath: string,
|
||||
schedule: string = "0 * * * *"
|
||||
): FileFixture {
|
||||
return {
|
||||
path: `${name}.schedule.yaml`,
|
||||
content: `path: "${name}"
|
||||
schedule: "${schedule}"
|
||||
script_path: "${scriptPath}"
|
||||
is_flow: false
|
||||
args: {}
|
||||
enabled: true
|
||||
timezone: "UTC"
|
||||
`,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,776 @@
|
||||
/**
|
||||
* Unified generate-metadata Command Tests
|
||||
*
|
||||
* Tests the new unified `generate-metadata` command that processes
|
||||
* scripts, flows, and apps together.
|
||||
*/
|
||||
|
||||
import { expect, test, describe } from "bun:test";
|
||||
import { withTestBackend } from "./test_backend.ts";
|
||||
import { addWorkspace } from "../workspace.ts";
|
||||
import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
||||
import {
|
||||
createLocalScript,
|
||||
createLocalFlow,
|
||||
createLocalApp,
|
||||
createLocalRawApp,
|
||||
createLocalScriptWithModules,
|
||||
} from "./test_fixtures.ts";
|
||||
|
||||
/**
|
||||
* Helper to set up a workspace with wmill.yaml
|
||||
*/
|
||||
async function setupWorkspace(
|
||||
backend: any,
|
||||
tempDir: string,
|
||||
workspaceName: string,
|
||||
nonDottedPaths = false
|
||||
) {
|
||||
const testWorkspace = {
|
||||
remote: backend.baseUrl,
|
||||
workspaceId: backend.workspace,
|
||||
name: workspaceName,
|
||||
token: backend.token
|
||||
};
|
||||
await addWorkspace(testWorkspace, { force: true, configDir: backend.testConfigDir });
|
||||
|
||||
await writeFile(`${tempDir}/wmill.yaml`, `defaultTs: bun
|
||||
${nonDottedPaths ? "nonDottedPaths: true\n" : ""}includes:
|
||||
- "**"
|
||||
excludes: []`, "utf-8");
|
||||
}
|
||||
|
||||
async function createLocalNonDottedFlow(tempDir: string, name: string) {
|
||||
const flowDir = `${tempDir}/f/test/${name}__flow`;
|
||||
await mkdir(flowDir, { recursive: true });
|
||||
|
||||
await writeFile(
|
||||
`${flowDir}/a.ts`,
|
||||
`export async function main() {\n return "Hello from flow ${name}";\n}`,
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
await writeFile(
|
||||
`${flowDir}/flow.yaml`,
|
||||
`summary: "${name} flow"
|
||||
description: "A flow for testing"
|
||||
value:
|
||||
modules:
|
||||
- id: a
|
||||
value:
|
||||
type: rawscript
|
||||
content: "!inline a.ts"
|
||||
language: bun
|
||||
input_transforms: {}
|
||||
schema:
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema"
|
||||
type: object
|
||||
properties: {}
|
||||
required: []
|
||||
`,
|
||||
"utf-8"
|
||||
);
|
||||
}
|
||||
|
||||
async function createLocalNonDottedApp(tempDir: string, name: string) {
|
||||
const appDir = `${tempDir}/f/test/${name}__app`;
|
||||
await mkdir(appDir, { recursive: true });
|
||||
|
||||
await writeFile(
|
||||
`${appDir}/app.yaml`,
|
||||
`summary: "${name} app"
|
||||
value:
|
||||
type: app
|
||||
grid:
|
||||
- id: button1
|
||||
data:
|
||||
type: buttoncomponent
|
||||
componentInput:
|
||||
type: runnable
|
||||
runnable:
|
||||
type: runnableByName
|
||||
inlineScript:
|
||||
content: |
|
||||
export async function main() {
|
||||
return "hello from app";
|
||||
}
|
||||
language: bun
|
||||
hiddenInlineScripts: []
|
||||
css: {}
|
||||
norefreshbar: false
|
||||
policy:
|
||||
on_behalf_of: null
|
||||
on_behalf_of_email: null
|
||||
triggerables: {}
|
||||
execution_mode: viewer
|
||||
`,
|
||||
"utf-8"
|
||||
);
|
||||
}
|
||||
|
||||
async function fileExists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// =============================================================================
|
||||
// Main test: processes scripts, flows, and apps together
|
||||
// =============================================================================
|
||||
|
||||
test("generate-metadata: processes scripts, flows, and apps together", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "unified_all_test");
|
||||
|
||||
// Create one of each type
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
await createLocalFlow(tempDir, "f/test", "my_flow");
|
||||
await createLocalApp(tempDir, "f/test", "my_app");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"unified_all_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
// Should find stale items
|
||||
expect(result.stdout).toContain("Found");
|
||||
expect(result.stdout).toContain("stale metadata");
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Flag tests
|
||||
// =============================================================================
|
||||
|
||||
describe("generate-metadata flags", () => {
|
||||
test("--includes filters to specific paths", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "includes_test");
|
||||
|
||||
// Create two scripts in different folders
|
||||
await createLocalScript(tempDir, "f/included", "script_a");
|
||||
await createLocalScript(tempDir, "f/excluded", "script_b");
|
||||
|
||||
// Run with --includes to only process f/included
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "-i", "f/included/**"],
|
||||
tempDir,
|
||||
"includes_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
// Should only mention the included script
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("script_a");
|
||||
expect(output).not.toContain("script_b");
|
||||
});
|
||||
});
|
||||
|
||||
test("--excludes filters out specific paths", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "excludes_test");
|
||||
|
||||
// Create two scripts
|
||||
await createLocalScript(tempDir, "f/keep", "script_keep");
|
||||
await createLocalScript(tempDir, "f/skip", "script_skip");
|
||||
|
||||
// Run with --excludes to skip f/skip
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "-e", "f/skip/**"],
|
||||
tempDir,
|
||||
"excludes_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("script_keep");
|
||||
expect(output).not.toContain("script_skip");
|
||||
});
|
||||
});
|
||||
|
||||
test("--dry-run shows stale items without updating", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "dry_run_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
|
||||
// Run with --dry-run
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"dry_run_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
// Should show stale items (Scripts section header)
|
||||
expect(result.stdout).toContain("Scripts");
|
||||
expect(result.stdout).toContain("my_script");
|
||||
// Should NOT show "Done" (didn't actually update)
|
||||
expect(result.stdout).not.toContain("Done");
|
||||
|
||||
// Run again without --dry-run to verify it would still be stale
|
||||
const result2 = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"dry_run_test"
|
||||
);
|
||||
expect(result2.stdout).toContain("Scripts");
|
||||
});
|
||||
});
|
||||
|
||||
test("--lock-only only regenerates locks", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "lock_only_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "--lock-only"],
|
||||
tempDir,
|
||||
"lock_only_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
test("--lock-only preserves non-dotted flow filenames", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "lock_only_non_dotted_test", true);
|
||||
|
||||
await createLocalNonDottedFlow(tempDir, "my_flow");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "--lock-only"],
|
||||
tempDir,
|
||||
"lock_only_non_dotted_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
|
||||
const flowDir = `${tempDir}/f/test/my_flow__flow`;
|
||||
const flowYaml = await readFile(`${flowDir}/flow.yaml`, "utf-8");
|
||||
|
||||
expect(flowYaml).toContain("!inline a.ts");
|
||||
expect(flowYaml).toContain("!inline a.lock");
|
||||
expect(flowYaml).not.toContain(".inline_script.");
|
||||
expect(await fileExists(`${flowDir}/a.lock`)).toEqual(true);
|
||||
expect(await fileExists(`${flowDir}/a.inline_script.ts`)).toEqual(false);
|
||||
expect(await fileExists(`${flowDir}/a.inline_script.lock`)).toEqual(false);
|
||||
});
|
||||
});
|
||||
|
||||
test("generate-metadata preserves non-dotted flow inline script filenames", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "full_gen_non_dotted_flow_test", true);
|
||||
|
||||
await createLocalNonDottedFlow(tempDir, "my_flow");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"full_gen_non_dotted_flow_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
|
||||
const flowDir = `${tempDir}/f/test/my_flow__flow`;
|
||||
const flowYaml = await readFile(`${flowDir}/flow.yaml`, "utf-8");
|
||||
|
||||
// Inline script references should use non-dotted naming
|
||||
expect(flowYaml).toContain("!inline a.ts");
|
||||
expect(flowYaml).toContain("!inline a.lock");
|
||||
expect(flowYaml).not.toContain(".inline_script.");
|
||||
expect(await fileExists(`${flowDir}/a.ts`)).toEqual(true);
|
||||
expect(await fileExists(`${flowDir}/a.lock`)).toEqual(true);
|
||||
expect(await fileExists(`${flowDir}/a.inline_script.ts`)).toEqual(false);
|
||||
expect(await fileExists(`${flowDir}/a.inline_script.lock`)).toEqual(false);
|
||||
});
|
||||
});
|
||||
|
||||
test("generate-metadata uses non-dotted app inline script filenames", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "non_dotted_app_gen_test", true);
|
||||
|
||||
await createLocalNonDottedApp(tempDir, "my_app");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"non_dotted_app_gen_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
|
||||
const appDir = `${tempDir}/f/test/my_app__app`;
|
||||
const appYaml = await readFile(`${appDir}/app.yaml`, "utf-8");
|
||||
|
||||
// Inline script references should use non-dotted naming
|
||||
expect(appYaml).not.toContain(".inline_script.");
|
||||
// Verify no dotted inline script files were created
|
||||
const { readdir: readdirAsync } = await import("node:fs/promises");
|
||||
const files = await readdirAsync(appDir);
|
||||
const dottedFiles = files.filter((f: string) => f.includes(".inline_script."));
|
||||
expect(dottedFiles.length).toEqual(0);
|
||||
});
|
||||
});
|
||||
|
||||
test("--schema-only only processes scripts (skips flows and apps)", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "schema_only_test");
|
||||
|
||||
// Create one of each type
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
await createLocalFlow(tempDir, "f/test", "my_flow");
|
||||
await createLocalApp(tempDir, "f/test", "my_app");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "--schema-only"],
|
||||
tempDir,
|
||||
"schema_only_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
// Should show "Checking scripts..." only
|
||||
expect(output).toContain("Checking scripts...");
|
||||
// Should find the script (Scripts section header)
|
||||
expect(output).toContain("Scripts");
|
||||
// Should NOT find flows or apps
|
||||
expect(output).not.toContain("Flows");
|
||||
expect(output).not.toContain("Apps");
|
||||
});
|
||||
});
|
||||
|
||||
test("--skip-scripts skips scripts", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "skip_scripts_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
await createLocalFlow(tempDir, "f/test", "my_flow");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "--skip-scripts"],
|
||||
tempDir,
|
||||
"skip_scripts_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
// Should NOT contain script
|
||||
expect(output).not.toContain("Scripts");
|
||||
// Should contain flow
|
||||
expect(output).toContain("Flows");
|
||||
});
|
||||
});
|
||||
|
||||
test("--skip-flows skips flows", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "skip_flows_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
await createLocalFlow(tempDir, "f/test", "my_flow");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "--skip-flows"],
|
||||
tempDir,
|
||||
"skip_flows_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
// Should contain script
|
||||
expect(output).toContain("Scripts");
|
||||
// Should NOT contain flow
|
||||
expect(output).not.toContain("Flows");
|
||||
});
|
||||
});
|
||||
|
||||
test("--skip-apps skips apps", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "skip_apps_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
await createLocalApp(tempDir, "f/test", "my_app");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "--skip-apps"],
|
||||
tempDir,
|
||||
"skip_apps_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
// Should contain script
|
||||
expect(output).toContain("Scripts");
|
||||
// Should NOT contain app
|
||||
expect(output).not.toContain("Apps");
|
||||
});
|
||||
});
|
||||
|
||||
test("shows 'All metadata up-to-date' when nothing to update", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "uptodate_test");
|
||||
|
||||
// Create a script and run generate-metadata twice
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
|
||||
// First run - generates metadata
|
||||
await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"uptodate_test"
|
||||
);
|
||||
|
||||
// Second run - should be up-to-date
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"uptodate_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout).toContain("up-to-date");
|
||||
});
|
||||
});
|
||||
|
||||
test("skipping all types shows warning", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "skip_all_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--skip-scripts", "--skip-flows", "--skip-apps"],
|
||||
tempDir,
|
||||
"skip_all_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout).toContain("Nothing to check");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Folder argument tests
|
||||
// =============================================================================
|
||||
|
||||
describe("generate-metadata folder argument", () => {
|
||||
test("filters to specific script folder", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "folder_script_test");
|
||||
|
||||
// Create scripts in different folders
|
||||
await createLocalScript(tempDir, "f/included", "script_a");
|
||||
await createLocalScript(tempDir, "f/excluded", "script_b");
|
||||
|
||||
// Run with folder argument
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/included/script_a.ts"],
|
||||
tempDir,
|
||||
"folder_script_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("script_a");
|
||||
expect(output).not.toContain("script_b");
|
||||
});
|
||||
});
|
||||
|
||||
test("filters to specific flow folder", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "folder_flow_test");
|
||||
|
||||
// Create flows in different folders
|
||||
await createLocalFlow(tempDir, "f/included", "flow_a");
|
||||
await createLocalFlow(tempDir, "f/excluded", "flow_b");
|
||||
|
||||
// Run with folder argument (flow folder path - uses .flow suffix by default)
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/included/flow_a.flow"],
|
||||
tempDir,
|
||||
"folder_flow_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("flow_a");
|
||||
expect(output).not.toContain("flow_b");
|
||||
});
|
||||
});
|
||||
|
||||
test("filters to specific app folder", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "folder_app_test");
|
||||
|
||||
// Create apps in different folders
|
||||
await createLocalApp(tempDir, "f/included", "app_a");
|
||||
await createLocalApp(tempDir, "f/excluded", "app_b");
|
||||
|
||||
// Run with folder argument (app folder path - uses .app suffix by default)
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/included/app_a.app"],
|
||||
tempDir,
|
||||
"folder_app_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("app_a");
|
||||
expect(output).not.toContain("app_b");
|
||||
});
|
||||
});
|
||||
|
||||
test("shows up-to-date when folder has no stale items", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "folder_uptodate_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
|
||||
// First run to generate metadata
|
||||
await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"folder_uptodate_test"
|
||||
);
|
||||
|
||||
// Second run with folder - should be up-to-date
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/test/my_script.ts"],
|
||||
tempDir,
|
||||
"folder_uptodate_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout).toContain("up-to-date");
|
||||
});
|
||||
});
|
||||
|
||||
test("trailing slash is stripped (matches deprecated behavior)", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "trailing_slash_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/test", "my_script");
|
||||
|
||||
// Run with trailing slash
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/test/my_script.ts/"],
|
||||
tempDir,
|
||||
"trailing_slash_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout).toContain("my_script");
|
||||
});
|
||||
});
|
||||
|
||||
test("parent folder matches all children", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "parent_folder_test");
|
||||
|
||||
// Create scripts in nested folders
|
||||
await createLocalScript(tempDir, "f/parent", "script_a");
|
||||
await createLocalScript(tempDir, "f/parent/child", "script_b");
|
||||
await createLocalScript(tempDir, "f/other", "script_c");
|
||||
|
||||
// Run with parent folder - should match both scripts in f/parent tree
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/parent"],
|
||||
tempDir,
|
||||
"parent_folder_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
expect(output).toContain("script_a");
|
||||
expect(output).toContain("script_b");
|
||||
expect(output).not.toContain("script_c");
|
||||
});
|
||||
});
|
||||
|
||||
test("non-existent folder shows up-to-date", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "nonexistent_folder_test");
|
||||
|
||||
await createLocalScript(tempDir, "f/exists", "my_script");
|
||||
|
||||
// Run with non-existent folder
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes", "f/does_not_exist"],
|
||||
tempDir,
|
||||
"nonexistent_folder_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
expect(result.stdout).toContain("up-to-date");
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// Scripts with modules
|
||||
// =============================================================================
|
||||
|
||||
describe("generate-metadata with script modules", () => {
|
||||
test("script with modules is detected as a single stale item", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "module_script_test");
|
||||
|
||||
// Create a script with module files
|
||||
await createLocalScriptWithModules(tempDir, "f/test", "order_workflow", "bun", [
|
||||
{ path: "helper.ts", content: 'export function validate() { return true; }\n' },
|
||||
{ path: "utils.ts", content: 'export const VERSION = "1.0";\n' },
|
||||
]);
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"module_script_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
// The main script should be listed as stale
|
||||
expect(output).toContain("order_workflow");
|
||||
// Module files should NOT appear as separate stale scripts (only within [changed modules: ...])
|
||||
const lines = output.split("\n");
|
||||
const staleLines = lines.filter((l: string) => l.includes("f/test/") || l.includes("f\\test\\"));
|
||||
expect(staleLines.length).toBe(1);
|
||||
expect(staleLines[0]).toContain("order_workflow");
|
||||
});
|
||||
});
|
||||
|
||||
test("module files are not treated as standalone scripts", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "module_not_standalone_test");
|
||||
|
||||
// Create a script with modules plus a regular script
|
||||
await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
|
||||
{ path: "helper.ts", content: 'export function greet() { return "hi"; }\n' },
|
||||
]);
|
||||
await createLocalScript(tempDir, "f/test", "standalone_script");
|
||||
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"module_not_standalone_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
const output = result.stdout + result.stderr;
|
||||
// Should list both the main script and standalone script
|
||||
expect(output).toContain("my_script");
|
||||
expect(output).toContain("standalone_script");
|
||||
// Should NOT list module helper as a separate entry
|
||||
// Count occurrences of "Scripts" header — there should be exactly one
|
||||
expect(output).toContain("Scripts");
|
||||
});
|
||||
});
|
||||
|
||||
test("script with modules generates metadata and becomes up-to-date", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "module_uptodate_test");
|
||||
|
||||
await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
|
||||
{ path: "helper.ts", content: 'export function greet() { return "hi"; }\n' },
|
||||
]);
|
||||
|
||||
// First run — should find stale items and generate metadata
|
||||
const result1 = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"module_uptodate_test"
|
||||
);
|
||||
expect(result1.code).toEqual(0);
|
||||
expect(result1.stdout).toContain("Done");
|
||||
|
||||
// Second run — should be up-to-date
|
||||
const result2 = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"module_uptodate_test"
|
||||
);
|
||||
expect(result2.code).toEqual(0);
|
||||
expect(result2.stdout).toContain("up-to-date");
|
||||
});
|
||||
});
|
||||
|
||||
test("modifying a module re-triggers stale detection", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "module_modify_test");
|
||||
|
||||
await createLocalScriptWithModules(tempDir, "f/test", "order_workflow", "bun", [
|
||||
{ path: "helper.ts", content: 'export function greet() { return "hi"; }\n' },
|
||||
{ path: "utils.ts", content: 'export const VERSION = "1.0";\n' },
|
||||
]);
|
||||
|
||||
// First run — generate metadata
|
||||
const result1 = await backend.runCLICommand(
|
||||
["generate-metadata", "--yes"],
|
||||
tempDir,
|
||||
"module_modify_test"
|
||||
);
|
||||
expect(result1.code).toEqual(0);
|
||||
|
||||
// Second run — should be up-to-date
|
||||
const result2 = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"module_modify_test"
|
||||
);
|
||||
expect(result2.code).toEqual(0);
|
||||
const output2 = result2.stdout + result2.stderr;
|
||||
expect(output2).not.toContain("order_workflow");
|
||||
|
||||
// Modify one module
|
||||
await writeFile(
|
||||
`${tempDir}/f/test/order_workflow__mod/helper.ts`,
|
||||
'export function greet() { return "hello world"; }\n',
|
||||
"utf-8"
|
||||
);
|
||||
|
||||
// Third run — should detect the script as stale with the changed module
|
||||
const result3 = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"module_modify_test"
|
||||
);
|
||||
expect(result3.code).toEqual(0);
|
||||
const output3 = result3.stdout + result3.stderr;
|
||||
expect(output3).toContain("order_workflow");
|
||||
});
|
||||
});
|
||||
|
||||
test("script with modules and lock files does not crash", async () => {
|
||||
await withTestBackend(async (backend, tempDir) => {
|
||||
await setupWorkspace(backend, tempDir, "module_with_locks_test");
|
||||
|
||||
// Create a script with modules that have lock files
|
||||
await createLocalScriptWithModules(tempDir, "f/test", "my_script", "bun", [
|
||||
{
|
||||
path: "helper.ts",
|
||||
content: 'import lodash from "lodash";\nexport const x = lodash.identity(1);\n',
|
||||
lock: "lodash@4.17.21\n",
|
||||
},
|
||||
]);
|
||||
|
||||
// Should not crash on lock files inside __mod/
|
||||
const result = await backend.runCLICommand(
|
||||
["generate-metadata", "--dry-run"],
|
||||
tempDir,
|
||||
"module_with_locks_test"
|
||||
);
|
||||
|
||||
expect(result.code).toEqual(0);
|
||||
// Should find the main script as stale, not crash on .lock files
|
||||
expect(result.stdout).toContain("my_script");
|
||||
});
|
||||
});
|
||||
});
|
||||
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "windmill-utils-internal",
|
||||
"version": "1.3.4",
|
||||
"version": "1.3.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "windmill-utils-internal",
|
||||
"version": "1.3.4",
|
||||
"version": "1.3.5",
|
||||
"license": "Apache 2.0",
|
||||
"devDependencies": {
|
||||
"@types/node": "^24.2.0",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "windmill-utils-internal",
|
||||
"version": "1.3.4",
|
||||
"version": "1.3.5",
|
||||
"description": "Internal utility functions for Windmill",
|
||||
"main": "dist/cjs/index.js",
|
||||
"module": "dist/esm/index.js",
|
||||
@@ -31,4 +31,4 @@
|
||||
"files": [
|
||||
"dist/**/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -150,8 +150,17 @@ export function extractInlineScripts(
|
||||
*/
|
||||
export function extractCurrentMapping(
|
||||
modules: FlowModule[] | undefined,
|
||||
mapping: Record<string, string> = {}
|
||||
mapping: Record<string, string> = {},
|
||||
failureModule?: FlowModule,
|
||||
preprocessorModule?: FlowModule,
|
||||
): Record<string, string> {
|
||||
if (failureModule) {
|
||||
extractCurrentMapping([failureModule], mapping);
|
||||
}
|
||||
if (preprocessorModule) {
|
||||
extractCurrentMapping([preprocessorModule], mapping);
|
||||
}
|
||||
|
||||
if (!modules || !Array.isArray(modules)) {
|
||||
return mapping;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
import { FlowModule, RawScript } from "../gen/types.gen.ts";
|
||||
import { AiAgent, FlowModule, FlowValue, RawScript } from "../gen/types.gen.ts";
|
||||
|
||||
export type LocalScriptInfo = {
|
||||
content: string;
|
||||
language: RawScript["language"];
|
||||
lock?: string;
|
||||
tag?: string;
|
||||
};
|
||||
|
||||
async function replaceRawscriptInline(
|
||||
id: string,
|
||||
@@ -118,4 +125,154 @@ export async function replaceInlineScripts(
|
||||
}));
|
||||
}
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces PathScript ("script" type) modules with RawScript ("rawscript" type) using local file content.
|
||||
* This is used during flow preview so that local script changes are tested instead of remote versions.
|
||||
*
|
||||
* @param modules - Array of flow modules to process
|
||||
* @param scriptReader - Function that takes a script path and returns local content/language/lock, or undefined if not found locally
|
||||
* @param logger - Logger for info/error messages
|
||||
*/
|
||||
export async function replacePathScriptsWithLocal(
|
||||
modules: FlowModule[],
|
||||
scriptReader: (scriptPath: string) => Promise<LocalScriptInfo | undefined>,
|
||||
logger: {
|
||||
info: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
} = {
|
||||
info: () => {},
|
||||
error: () => {},
|
||||
}
|
||||
): Promise<void> {
|
||||
await Promise.all(modules.map(async (module) => {
|
||||
if (!module.value) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (module.value.type === "script") {
|
||||
const scriptPath = module.value.path;
|
||||
const localScript = await scriptReader(scriptPath);
|
||||
if (localScript) {
|
||||
const pathScript = module.value;
|
||||
module.value = {
|
||||
type: "rawscript",
|
||||
content: localScript.content,
|
||||
language: localScript.language,
|
||||
lock: localScript.lock,
|
||||
path: scriptPath,
|
||||
input_transforms: pathScript.input_transforms,
|
||||
tag: pathScript.tag_override ?? localScript.tag,
|
||||
} satisfies RawScript;
|
||||
}
|
||||
} else if (module.value.type === "forloopflow" || module.value.type === "whileloopflow") {
|
||||
await replacePathScriptsWithLocal(module.value.modules, scriptReader, logger);
|
||||
} else if (module.value.type === "branchall") {
|
||||
await Promise.all(module.value.branches.map(async (branch) => {
|
||||
await replacePathScriptsWithLocal(branch.modules, scriptReader, logger);
|
||||
}));
|
||||
} else if (module.value.type === "branchone") {
|
||||
await Promise.all(module.value.branches.map(async (branch) => {
|
||||
await replacePathScriptsWithLocal(branch.modules, scriptReader, logger);
|
||||
}));
|
||||
await replacePathScriptsWithLocal(module.value.default, scriptReader, logger);
|
||||
} else if (module.value.type === "aiagent") {
|
||||
await Promise.all((module.value.tools ?? []).map(async (tool) => {
|
||||
const toolValue = tool.value;
|
||||
if (!toolValue || toolValue.tool_type !== "flowmodule" || toolValue.type !== "script") {
|
||||
return;
|
||||
}
|
||||
const localScript = await scriptReader(toolValue.path);
|
||||
if (localScript) {
|
||||
(tool as AiAgent["tools"][number]).value = {
|
||||
tool_type: "flowmodule",
|
||||
type: "rawscript",
|
||||
content: localScript.content,
|
||||
language: localScript.language,
|
||||
lock: localScript.lock,
|
||||
path: toolValue.path,
|
||||
input_transforms: toolValue.input_transforms,
|
||||
tag: toolValue.tag_override ?? localScript.tag,
|
||||
};
|
||||
}
|
||||
}));
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
function collectPathScriptPathsFromModules(
|
||||
modules: FlowModule[],
|
||||
paths: Set<string>
|
||||
): void {
|
||||
for (const module of modules) {
|
||||
if (!module.value) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (module.value.type === "script") {
|
||||
paths.add(module.value.path);
|
||||
} else if (
|
||||
module.value.type === "forloopflow" ||
|
||||
module.value.type === "whileloopflow"
|
||||
) {
|
||||
collectPathScriptPathsFromModules(module.value.modules, paths);
|
||||
} else if (module.value.type === "branchall") {
|
||||
for (const branch of module.value.branches) {
|
||||
collectPathScriptPathsFromModules(branch.modules, paths);
|
||||
}
|
||||
} else if (module.value.type === "branchone") {
|
||||
for (const branch of module.value.branches) {
|
||||
collectPathScriptPathsFromModules(branch.modules, paths);
|
||||
}
|
||||
collectPathScriptPathsFromModules(module.value.default, paths);
|
||||
} else if (module.value.type === "aiagent") {
|
||||
for (const tool of module.value.tools ?? []) {
|
||||
const toolValue = tool.value;
|
||||
if (
|
||||
toolValue &&
|
||||
toolValue.tool_type === "flowmodule" &&
|
||||
toolValue.type === "script"
|
||||
) {
|
||||
paths.add(toolValue.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Replaces all PathScript modules in a flow value (modules, failure_module, preprocessor_module)
|
||||
* with RawScript using local file content.
|
||||
*/
|
||||
export async function replaceAllPathScriptsWithLocal(
|
||||
flowValue: FlowValue,
|
||||
scriptReader: (scriptPath: string) => Promise<LocalScriptInfo | undefined>,
|
||||
logger: {
|
||||
info: (message: string) => void;
|
||||
error: (message: string) => void;
|
||||
} = {
|
||||
info: () => {},
|
||||
error: () => {},
|
||||
}
|
||||
): Promise<void> {
|
||||
await replacePathScriptsWithLocal(flowValue.modules, scriptReader, logger);
|
||||
if (flowValue.failure_module) {
|
||||
await replacePathScriptsWithLocal([flowValue.failure_module], scriptReader, logger);
|
||||
}
|
||||
if (flowValue.preprocessor_module) {
|
||||
await replacePathScriptsWithLocal([flowValue.preprocessor_module], scriptReader, logger);
|
||||
}
|
||||
}
|
||||
|
||||
export function collectPathScriptPaths(flowValue: FlowValue): string[] {
|
||||
const paths = new Set<string>();
|
||||
collectPathScriptPathsFromModules(flowValue.modules, paths);
|
||||
if (flowValue.failure_module) {
|
||||
collectPathScriptPathsFromModules([flowValue.failure_module], paths);
|
||||
}
|
||||
if (flowValue.preprocessor_module) {
|
||||
collectPathScriptPathsFromModules([flowValue.preprocessor_module], paths);
|
||||
}
|
||||
return [...paths];
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user