mirror of
https://github.com/windmill-labs/windmill.git
synced 2026-09-10 08:07:03 +00:00
Merge remote-tracking branch 'origin/main' into fork-datatable-schema-export
This commit is contained in:
Diego Imbert
@@ -7,6 +7,7 @@ import { yamlParseFile } from "../../utils/yaml.ts";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { GlobalOptions } from "../../types.ts";
|
||||
import {
|
||||
readLockfile,
|
||||
checkifMetadataUptodate,
|
||||
blueColor,
|
||||
clearGlobalLock,
|
||||
@@ -41,6 +42,8 @@ import { mergeConfigWithConfigFile, SyncOptions } from "../../core/conf.ts";
|
||||
import { resolveWorkspace } from "../../core/context.ts";
|
||||
import { requireLogin } from "../../core/auth.ts";
|
||||
import { getNonDottedPaths } from "../../utils/resource_folders.ts";
|
||||
import { extractRelativeImports } from "../../utils/relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts";
|
||||
|
||||
const TOP_HASH = "__app_hash";
|
||||
export const APP_BACKEND_FOLDER = "backend";
|
||||
@@ -93,7 +96,16 @@ async function generateAppHash(
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates locks for inline scripts in an app
|
||||
* Result of generating app locks, including which scripts were updated
|
||||
*/
|
||||
export interface AppLocksResult {
|
||||
path: string;
|
||||
updatedScripts: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates locks for inline scripts in an app.
|
||||
* Returns the path if dry-run, or AppLocksResult with updated scripts if actual update occurred.
|
||||
*/
|
||||
export async function generateAppLocksInternal(
|
||||
appFolder: string,
|
||||
@@ -104,8 +116,10 @@ export async function generateAppLocksInternal(
|
||||
defaultTs?: "bun" | "deno";
|
||||
},
|
||||
justUpdateMetadataLock?: boolean,
|
||||
noStaleMessage?: boolean
|
||||
): Promise<string | void> {
|
||||
noStaleMessage?: boolean,
|
||||
legacyBehaviour?: boolean,
|
||||
tree?: DoubleLinkedDependencyTree
|
||||
): Promise<string | AppLocksResult | void> {
|
||||
if (appFolder.endsWith(SEP)) {
|
||||
appFolder = appFolder.substring(0, appFolder.length - 1);
|
||||
}
|
||||
@@ -116,9 +130,6 @@ export async function generateAppLocksInternal(
|
||||
log.info(`Generating locks for app ${appFolder} at ${remote_path}`);
|
||||
}
|
||||
|
||||
const rawWorkspaceDependencies: Record<string, string> =
|
||||
await getRawWorkspaceDependencies();
|
||||
|
||||
// Read the app file first to filter workspace dependencies
|
||||
const appFilePath = path.join(
|
||||
appFolder,
|
||||
@@ -126,35 +137,80 @@ export async function generateAppLocksInternal(
|
||||
);
|
||||
const appFile = (await yamlParseFile(appFilePath)) as AppFile;
|
||||
|
||||
// Filter workspace dependencies based on inline scripts' languages and annotations
|
||||
const appValue = rawApp ? (appFile as RawAppFile).runnables : (appFile as NormalAppFile).value;
|
||||
const filteredDeps = await filterWorkspaceDependenciesForApp(
|
||||
appValue,
|
||||
rawWorkspaceDependencies,
|
||||
appFolder
|
||||
);
|
||||
const folderNormalized = appFolder.replaceAll(SEP, "/");
|
||||
|
||||
let hashes = await generateAppHash(
|
||||
filteredDeps,
|
||||
appFolder,
|
||||
rawApp,
|
||||
opts.defaultTs
|
||||
);
|
||||
let filteredDeps: Record<string, string> = {};
|
||||
const conf = await readLockfile();
|
||||
|
||||
const conf = await import("../../utils/metadata.ts").then((m) =>
|
||||
m.readLockfile()
|
||||
);
|
||||
if (
|
||||
await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH)
|
||||
) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
// New behaviour: tree-based dependency tracking
|
||||
if (!legacyBehaviour && tree) {
|
||||
if (dryRun) {
|
||||
const hashes = await generateAppHash({}, appFolder, rawApp, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
// For raw apps in new format, runnables are in separate files under backend/
|
||||
let treeAppValue = structuredClone(appValue);
|
||||
if (rawApp) {
|
||||
const runnablesPath = path.join(appFolder, APP_BACKEND_FOLDER);
|
||||
const runnablesFromFiles = await loadRunnablesFromBackend(runnablesPath);
|
||||
if (Object.keys(runnablesFromFiles).length > 0) {
|
||||
treeAppValue = runnablesFromFiles;
|
||||
}
|
||||
}
|
||||
|
||||
// First pass: add inline scripts as separate nodes, then add app node importing them
|
||||
const inlineScriptPaths: string[] = [];
|
||||
await traverseAndProcessInlineScripts(treeAppValue, async (inlineScript, context) => {
|
||||
if (!inlineScript.content || !inlineScript.language) {
|
||||
return inlineScript;
|
||||
}
|
||||
|
||||
let content = inlineScript.content;
|
||||
// Resolve !inline references
|
||||
if (typeof content === "string" && content.startsWith("!inline ")) {
|
||||
const filePath = appFolder + SEP + content.replace("!inline ", "");
|
||||
try {
|
||||
content = await readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
return inlineScript;
|
||||
}
|
||||
}
|
||||
|
||||
const treePath = folderNormalized + "/" + context.path.join("/");
|
||||
const language = inlineScript.language as ScriptLanguage;
|
||||
const imports = await extractRelativeImports(content, treePath, language);
|
||||
await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, appFolder, false);
|
||||
inlineScriptPaths.push(treePath);
|
||||
|
||||
return inlineScript;
|
||||
});
|
||||
|
||||
await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "app", folderNormalized, appFolder, isDirectlyStale, rawApp);
|
||||
return;
|
||||
}
|
||||
// Second pass: get mismatched workspace deps from tree
|
||||
// TODO: pass raw workspace deps more precisely to every inline script lock generation call
|
||||
// (currently we pass the union of all mismatched deps filtered for the whole app)
|
||||
filteredDeps = await filterWorkspaceDependenciesForApp(appValue, tree.getMismatchedWorkspaceDeps(), appFolder);
|
||||
} else {
|
||||
// Legacy behaviour
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
filteredDeps = await filterWorkspaceDependenciesForApp(appValue, rawWorkspaceDependencies, appFolder);
|
||||
|
||||
const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(appFolder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
if (!isDirectlyStale) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`App ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
|
||||
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
|
||||
@@ -167,7 +223,11 @@ export async function generateAppLocksInternal(
|
||||
);
|
||||
}
|
||||
|
||||
let updatedScripts: string[] = [];
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
const hashes = await generateAppHash(filteredDeps, appFolder, rawApp, opts.defaultTs);
|
||||
|
||||
const changedScripts = [];
|
||||
// Find hashes that do not correspond to previous hashes
|
||||
for (const [scriptPath, hash] of Object.entries(hashes)) {
|
||||
@@ -179,7 +239,13 @@ export async function generateAppLocksInternal(
|
||||
}
|
||||
}
|
||||
|
||||
if (changedScripts.length > 0) {
|
||||
// Get temp_script_refs from tree for relative import resolution
|
||||
const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized);
|
||||
|
||||
// In tree mode, the tree already verified this app is stale (possibly via dependency change).
|
||||
// Per-script hashes only detect content changes, not transitive dependency changes,
|
||||
// so we must regenerate locks for all inline scripts regardless.
|
||||
if (changedScripts.length > 0 || (tree && !legacyBehaviour)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
`Recomputing locks of ${changedScripts.join(", ")} in ${appFolder}`
|
||||
@@ -201,13 +267,15 @@ export async function generateAppLocksInternal(
|
||||
replaceInlineScripts(runnables, runnablesPath + SEP, false);
|
||||
|
||||
// Update the app runnables with new locks (writes to separate files)
|
||||
await updateRawAppRunnables(
|
||||
updatedScripts = await updateRawAppRunnables(
|
||||
workspace,
|
||||
runnables,
|
||||
remote_path,
|
||||
appFolder,
|
||||
filteredDeps,
|
||||
opts.defaultTs
|
||||
opts.defaultTs,
|
||||
noStaleMessage,
|
||||
tempScriptRefs
|
||||
);
|
||||
// Note: updateRawAppRunnables now writes each runnable to its own file
|
||||
} else {
|
||||
@@ -217,14 +285,18 @@ export async function generateAppLocksInternal(
|
||||
replaceInlineScripts(normalAppFile.value, appFolder + SEP, false);
|
||||
|
||||
// Update the app value with new locks
|
||||
normalAppFile.value = await updateAppInlineScripts(
|
||||
const result = await updateAppInlineScripts(
|
||||
workspace,
|
||||
normalAppFile.value,
|
||||
remote_path,
|
||||
appFolder,
|
||||
filteredDeps,
|
||||
opts.defaultTs
|
||||
opts.defaultTs,
|
||||
noStaleMessage,
|
||||
tempScriptRefs
|
||||
);
|
||||
normalAppFile.value = result.value;
|
||||
updatedScripts = result.updatedScripts;
|
||||
|
||||
// Write the updated app file (only for normal apps, raw apps use separate files)
|
||||
writeIfChanged(
|
||||
@@ -237,20 +309,23 @@ export async function generateAppLocksInternal(
|
||||
}
|
||||
}
|
||||
|
||||
// Regenerate hashes after updates
|
||||
hashes = await generateAppHash(
|
||||
filteredDeps,
|
||||
// Non-legacy mode excludes workspace deps from hash (tracked via tree instead)
|
||||
const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps;
|
||||
const finalHashes = await generateAppHash(
|
||||
depsForHash,
|
||||
appFolder,
|
||||
rawApp,
|
||||
opts.defaultTs
|
||||
);
|
||||
await clearGlobalLock(appFolder);
|
||||
for (const [scriptPath, hash] of Object.entries(hashes)) {
|
||||
for (const [scriptPath, hash] of Object.entries(finalHashes)) {
|
||||
await updateMetadataGlobalLock(appFolder, hash, scriptPath);
|
||||
}
|
||||
if (!noStaleMessage) {
|
||||
log.info(colors.green(`App ${remote_path} lockfiles updated`));
|
||||
}
|
||||
|
||||
return { path: remote_path, updatedScripts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -340,6 +415,7 @@ async function traverseAndProcessInlineScripts(
|
||||
* Updates locks for all runnables in a raw app, generating locks inline script by inline script.
|
||||
* Writes each runnable to its own YAML file in the backend folder (new format).
|
||||
* Also writes content and lock files to the runnables folder.
|
||||
* Returns the list of runnable IDs that had their locks updated.
|
||||
*/
|
||||
async function updateRawAppRunnables(
|
||||
workspace: Workspace,
|
||||
@@ -347,8 +423,11 @@ async function updateRawAppRunnables(
|
||||
remotePath: string,
|
||||
appFolder: string,
|
||||
rawDeps?: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" = "bun"
|
||||
): Promise<void> {
|
||||
defaultTs: "bun" | "deno" = "bun",
|
||||
noStaleMessage?: boolean,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string[]> {
|
||||
const updatedRunnables: string[] = [];
|
||||
const runnablesFolder = path.join(appFolder, APP_BACKEND_FOLDER);
|
||||
|
||||
// Ensure runnables folder exists
|
||||
@@ -414,12 +493,11 @@ async function updateRawAppRunnables(
|
||||
continue;
|
||||
}
|
||||
|
||||
log.info(
|
||||
colors.gray(
|
||||
`Generating lock for runnable ${runnableId} (${language})
|
||||
}`
|
||||
)
|
||||
);
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(`Generating lock for runnable ${runnableId} (${language})`)
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const lock = await generateInlineScriptLock(
|
||||
@@ -427,7 +505,8 @@ async function updateRawAppRunnables(
|
||||
content,
|
||||
language,
|
||||
`${remotePath}/${runnableId}`,
|
||||
rawDeps
|
||||
rawDeps,
|
||||
tempScriptRefs
|
||||
);
|
||||
|
||||
// Determine file extension for this language
|
||||
@@ -459,11 +538,15 @@ async function updateRawAppRunnables(
|
||||
// Write the runnable to its own YAML file
|
||||
writeRunnableToBackend(runnablesFolder, runnableId, simplifiedRunnable);
|
||||
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
updatedRunnables.push(runnableId);
|
||||
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${runnableId}.yaml, ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
}
|
||||
} catch (error: any) {
|
||||
log.error(
|
||||
colors.red(
|
||||
@@ -474,11 +557,14 @@ async function updateRawAppRunnables(
|
||||
writeRunnableToBackend(runnablesFolder, runnableId, runnable);
|
||||
}
|
||||
}
|
||||
|
||||
return updatedRunnables;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates locks for all inline scripts in a normal app, similar to updateRawAppRunnables
|
||||
* but for the app.value structure instead of app.runnables
|
||||
* but for the app.value structure instead of app.runnables.
|
||||
* Returns a tuple of [updated app value, list of script names that were updated].
|
||||
*/
|
||||
async function updateAppInlineScripts(
|
||||
workspace: Workspace,
|
||||
@@ -486,9 +572,12 @@ async function updateAppInlineScripts(
|
||||
remotePath: string,
|
||||
appFolder: string,
|
||||
rawDeps?: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" = "bun"
|
||||
): Promise<any> {
|
||||
defaultTs: "bun" | "deno" = "bun",
|
||||
noStaleMessage?: boolean,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<{ value: any; updatedScripts: string[] }> {
|
||||
const pathAssigner = newPathAssigner(defaultTs, { skipInlineScriptSuffix: getNonDottedPaths() });
|
||||
const updatedScripts: string[] = [];
|
||||
|
||||
const processor: InlineScriptProcessor = async (inlineScript, context) => {
|
||||
const language = inlineScript.language as SupportedLanguage;
|
||||
@@ -518,20 +607,23 @@ async function updateAppInlineScripts(
|
||||
try {
|
||||
let lock: string | undefined;
|
||||
if (language !== "frontend") {
|
||||
log.info(
|
||||
colors.gray(
|
||||
`Generating lock for inline script "${scriptName}" at ${context.path.join(
|
||||
"."
|
||||
)} (${language})`
|
||||
)
|
||||
);
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(
|
||||
`Generating lock for inline script "${scriptName}" at ${context.path.join(
|
||||
"."
|
||||
)} (${language})`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
lock = await generateInlineScriptLock(
|
||||
workspace,
|
||||
content,
|
||||
language,
|
||||
scriptPath,
|
||||
rawDeps
|
||||
rawDeps,
|
||||
tempScriptRefs
|
||||
);
|
||||
}
|
||||
// Determine file extension for this language (following extractInlineScriptsForApps pattern)
|
||||
@@ -553,11 +645,18 @@ async function updateAppInlineScripts(
|
||||
const inlineLockRef =
|
||||
lock && lock !== "" ? `!inline ${basePath}lock` : "";
|
||||
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.gray(
|
||||
` Written ${basePath}${ext}${lock ? ` and ${basePath}lock` : ""}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
// Track that this script was updated (only for non-frontend scripts that needed locks)
|
||||
if (language !== "frontend") {
|
||||
updatedScripts.push(scriptName);
|
||||
}
|
||||
|
||||
return {
|
||||
...inlineScript,
|
||||
@@ -577,7 +676,8 @@ async function updateAppInlineScripts(
|
||||
}
|
||||
};
|
||||
|
||||
return await traverseAndProcessInlineScripts(appValue, processor);
|
||||
const updatedValue = await traverseAndProcessInlineScripts(appValue, processor);
|
||||
return { value: updatedValue, updatedScripts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -588,7 +688,8 @@ async function generateInlineScriptLock(
|
||||
content: string,
|
||||
language: string,
|
||||
scriptPath: string,
|
||||
rawWorkspaceDependencies: Record<string, string> | undefined
|
||||
rawWorkspaceDependencies: Record<string, string> | undefined,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string> {
|
||||
// Filter workspace dependencies to only include those matching this script's language and annotations
|
||||
const filteredDeps = rawWorkspaceDependencies
|
||||
@@ -619,6 +720,9 @@ async function generateInlineScriptLock(
|
||||
? filteredDeps
|
||||
: null,
|
||||
entrypoint: scriptPath,
|
||||
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? { temp_script_refs: tempScriptRefs }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -16,33 +16,34 @@ import { resolveWorkspace } from "../../core/context.ts";
|
||||
import {
|
||||
SyncOptions,
|
||||
mergeConfigWithConfigFile,
|
||||
readConfigFile,
|
||||
} from "../../core/conf.ts";
|
||||
import { exts, removeExtensionToPath } from "../script/script.ts";
|
||||
import { inferContentTypeFromFilePath } from "../../utils/script_common.ts";
|
||||
import { OpenFlow } from "../../../gen/types.gen.ts";
|
||||
import { FlowFile } from "../flow/flow.ts";
|
||||
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { replaceInlineScripts, replaceAllPathScriptsWithLocal } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { parseMetadataFile } from "../../utils/metadata.ts";
|
||||
import {
|
||||
getFolderSuffixWithSep,
|
||||
getMetadataFileName,
|
||||
extractFolderPath,
|
||||
} from "../../utils/resource_folders.ts";
|
||||
import { listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import { createPreviewLocalScriptReader } from "../../utils/local_path_scripts.ts";
|
||||
|
||||
const PORT = 3001;
|
||||
async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
|
||||
log.info("Started dev mode");
|
||||
const conf = await readConfigFile();
|
||||
let currentLastEdit: LastEditScript | LastEditFlow | undefined = undefined;
|
||||
|
||||
const fsWatcher = watch(".", { recursive: true });
|
||||
const base = await realpath(".");
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
const ignore = await ignoreF(opts);
|
||||
const codebases = await listSyncCodebases(opts);
|
||||
|
||||
const changesTimeouts: Record<string, ReturnType<typeof setTimeout>> = {};
|
||||
function watchChanges() {
|
||||
@@ -56,7 +57,11 @@ async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
}
|
||||
changesTimeouts[key] = setTimeout(async () => {
|
||||
delete changesTimeouts[key];
|
||||
await loadPaths([filePath]);
|
||||
await loadPaths([filePath]).catch((error) => {
|
||||
log.error(
|
||||
`Failed to reload ${filePath}: ${error instanceof Error ? error.message : error}`
|
||||
);
|
||||
});
|
||||
}, 100);
|
||||
});
|
||||
fsWatcher.on("error", (err) => {
|
||||
@@ -94,6 +99,13 @@ async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
SEP,
|
||||
undefined,
|
||||
);
|
||||
// Replace PathScript modules with local file content so dev mode uses local versions
|
||||
const localScriptReader = createPreviewLocalScriptReader({
|
||||
exts,
|
||||
defaultTs: opts.defaultTs,
|
||||
codebases,
|
||||
});
|
||||
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
|
||||
currentLastEdit = {
|
||||
type: "flow",
|
||||
flow: localFlow,
|
||||
@@ -105,7 +117,7 @@ async function dev(opts: GlobalOptions & SyncOptions) {
|
||||
const content = await readFile(cpath, "utf-8");
|
||||
const splitted = cpath.split(".");
|
||||
const wmPath = splitted[0];
|
||||
const lang = inferContentTypeFromFilePath(cpath, conf.defaultTs);
|
||||
const lang = inferContentTypeFromFilePath(cpath, opts.defaultTs);
|
||||
const typed =
|
||||
(await parseMetadataFile(
|
||||
removeExtensionToPath(cpath),
|
||||
|
||||
@@ -18,8 +18,19 @@ import { defaultFlowDefinition } from "../../../bootstrap/flow_bootstrap.ts";
|
||||
import { SyncOptions, mergeConfigWithConfigFile } from "../../core/conf.ts";
|
||||
import { FSFSElement, elementsToMap, ignoreF } from "../sync/sync.ts";
|
||||
import { Flow } from "../../../gen/types.gen.ts";
|
||||
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import {
|
||||
collectPathScriptPaths,
|
||||
replaceInlineScripts,
|
||||
replaceAllPathScriptsWithLocal,
|
||||
} from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { generateFlowLockInternal } from "./flow_metadata.ts";
|
||||
import { exts } from "../script/script.ts";
|
||||
import type { SyncCodebase } from "../../utils/codebase.ts";
|
||||
import { listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import {
|
||||
createPreviewLocalScriptReader,
|
||||
resolvePreviewLocalScriptState,
|
||||
} from "../../utils/local_path_scripts.ts";
|
||||
|
||||
export interface FlowFile {
|
||||
summary: string;
|
||||
@@ -28,6 +39,90 @@ export interface FlowFile {
|
||||
schema?: any;
|
||||
}
|
||||
|
||||
function normalizeOptionalString(value: string | null | undefined): string | undefined {
|
||||
return typeof value === "string" && value.trim() === "" ? undefined : value ?? undefined;
|
||||
}
|
||||
|
||||
function normalizeComparableContent(value: string | undefined): string | undefined {
|
||||
return value?.replaceAll("\r\n", "\n").replace(/\n$/, "");
|
||||
}
|
||||
|
||||
async function findDivergedLocalPathScripts(
|
||||
workspaceId: string,
|
||||
scriptPaths: string[],
|
||||
opts: {
|
||||
exts: string[];
|
||||
defaultTs?: "bun" | "deno";
|
||||
codebases: SyncCodebase[];
|
||||
}
|
||||
): Promise<{ changed: string[]; missing: string[] }> {
|
||||
const changed: string[] = [];
|
||||
const missing: string[] = [];
|
||||
|
||||
for (const scriptPath of scriptPaths) {
|
||||
const localScript = await resolvePreviewLocalScriptState(scriptPath, opts);
|
||||
if (!localScript) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let remoteScript;
|
||||
try {
|
||||
remoteScript = await wmill.getScriptByPath({
|
||||
workspace: workspaceId,
|
||||
path: scriptPath,
|
||||
});
|
||||
} catch {
|
||||
missing.push(scriptPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
const remoteLock = normalizeOptionalString(remoteScript.lock);
|
||||
const diverged =
|
||||
normalizeComparableContent(localScript.content) !==
|
||||
normalizeComparableContent(remoteScript.content) ||
|
||||
localScript.language !== remoteScript.language ||
|
||||
(localScript.lock !== undefined &&
|
||||
normalizeComparableContent(localScript.lock) !==
|
||||
normalizeComparableContent(remoteLock)) ||
|
||||
localScript.tag !== normalizeOptionalString(remoteScript.tag) ||
|
||||
localScript.codebaseDigest !== normalizeOptionalString(remoteScript.codebase);
|
||||
|
||||
if (diverged) {
|
||||
changed.push(scriptPath);
|
||||
}
|
||||
}
|
||||
|
||||
return { changed, missing };
|
||||
}
|
||||
|
||||
function warnAboutLocalPathScriptDivergence(
|
||||
divergence: { changed: string[]; missing: string[] }
|
||||
): void {
|
||||
if (divergence.changed.length === 0 && divergence.missing.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const details: string[] = [];
|
||||
if (divergence.changed.length > 0) {
|
||||
details.push(
|
||||
`These workspace scripts differ from the deployed version:\n${divergence.changed
|
||||
.map((path) => `- ${path}`)
|
||||
.join("\n")}`
|
||||
);
|
||||
}
|
||||
if (divergence.missing.length > 0) {
|
||||
details.push(
|
||||
`These scripts do not exist in the workspace yet:\n${divergence.missing
|
||||
.map((path) => `- ${path}`)
|
||||
.join("\n")}`
|
||||
);
|
||||
}
|
||||
|
||||
log.warn(
|
||||
`Using local PathScript files for flow preview.\n${details.join("\n")}\nUse --remote to preview deployed workspace scripts instead.`
|
||||
);
|
||||
}
|
||||
|
||||
const alreadySynced: string[] = [];
|
||||
|
||||
export async function pushFlow(
|
||||
@@ -233,11 +328,17 @@ async function preview(
|
||||
opts: GlobalOptions & {
|
||||
data?: string;
|
||||
silent: boolean;
|
||||
},
|
||||
remote?: boolean;
|
||||
} & SyncOptions,
|
||||
flowPath: string
|
||||
) {
|
||||
const useLocalPathScripts = !opts.remote;
|
||||
if (useLocalPathScripts) {
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
}
|
||||
const workspace = await resolveWorkspace(opts);
|
||||
await requireLogin(opts);
|
||||
const codebases = useLocalPathScripts ? listSyncCodebases(opts) : [];
|
||||
|
||||
// Normalize path - ensure it's a directory path to a .flow folder
|
||||
if (!flowPath.endsWith(".flow") && !flowPath.endsWith(".flow" + SEP)) {
|
||||
@@ -274,6 +375,31 @@ async function preview(
|
||||
await replaceInlineScripts([localFlow.value.preprocessor_module], fileReader, log, flowPath, SEP);
|
||||
}
|
||||
|
||||
if (useLocalPathScripts) {
|
||||
const scriptPaths = collectPathScriptPaths(localFlow.value);
|
||||
if (scriptPaths.length > 0) {
|
||||
const divergence = await findDivergedLocalPathScripts(
|
||||
workspace.workspaceId,
|
||||
scriptPaths,
|
||||
{
|
||||
exts,
|
||||
defaultTs: opts.defaultTs,
|
||||
codebases,
|
||||
}
|
||||
);
|
||||
if (!opts.silent) {
|
||||
warnAboutLocalPathScriptDivergence(divergence);
|
||||
}
|
||||
}
|
||||
|
||||
const localScriptReader = createPreviewLocalScriptReader({
|
||||
exts,
|
||||
defaultTs: opts.defaultTs,
|
||||
codebases,
|
||||
});
|
||||
await replaceAllPathScriptsWithLocal(localFlow.value, localScriptReader, log);
|
||||
}
|
||||
|
||||
const input = opts.data ? await resolve(opts.data) : {};
|
||||
|
||||
if (!opts.silent) {
|
||||
@@ -311,6 +437,7 @@ async function preview(
|
||||
export async function generateLocks(
|
||||
opts: GlobalOptions & {
|
||||
yes?: boolean;
|
||||
dryRun?: boolean;
|
||||
} & SyncOptions,
|
||||
folder: string | undefined
|
||||
) {
|
||||
@@ -361,6 +488,10 @@ export async function generateLocks(
|
||||
}
|
||||
|
||||
if (hasAny) {
|
||||
if (opts.dryRun) {
|
||||
log.info(colors.gray("Dry run complete."));
|
||||
return;
|
||||
}
|
||||
if (
|
||||
!opts.yes &&
|
||||
!(await Confirm.prompt({
|
||||
@@ -444,7 +575,7 @@ const command = new Command()
|
||||
.action(run as any)
|
||||
.command(
|
||||
"preview",
|
||||
"preview a local flow without deploying it. Runs the flow definition from local files."
|
||||
"preview a local flow without deploying it. Runs the flow definition from local files and uses local PathScripts by default."
|
||||
)
|
||||
.arguments("<flow_path:string>")
|
||||
.option(
|
||||
@@ -455,6 +586,10 @@ const command = new Command()
|
||||
"-s --silent",
|
||||
"Do not output anything other then the final output. Useful for scripting."
|
||||
)
|
||||
.option(
|
||||
"--remote",
|
||||
"Use deployed workspace scripts for PathScript steps instead of local files."
|
||||
)
|
||||
.action(preview as any)
|
||||
.command(
|
||||
"generate-locks",
|
||||
@@ -462,6 +597,7 @@ const command = new Command()
|
||||
)
|
||||
.arguments("[flow:file]")
|
||||
.option("--yes", "Skip confirmation prompt")
|
||||
.option("--dry-run", "Perform a dry run without making changes")
|
||||
.option(
|
||||
"-i --includes <patterns:file[]>",
|
||||
"Comma separated patterns to specify which file to take into account (among files that are compatible with windmill). Patterns can include * (any string until '/') and ** (any string)"
|
||||
|
||||
@@ -29,7 +29,9 @@ import { FlowFile } from "./flow.ts";
|
||||
import { FlowValue } from "../../../gen/types.gen.ts";
|
||||
import { replaceInlineScripts } from "../../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import { workspaceDependenciesLanguages } from "../../utils/script_common.ts";
|
||||
import { extractNameFromFolder, getFolderSuffix } from "../../utils/resource_folders.ts";
|
||||
import { extractNameFromFolder, getFolderSuffix, getNonDottedPaths } from "../../utils/resource_folders.ts";
|
||||
import { extractRelativeImports } from "../../utils/relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "../../utils/dependency_tree.ts";
|
||||
|
||||
const TOP_HASH = "__flow_hash";
|
||||
async function generateFlowHash(
|
||||
@@ -51,6 +53,14 @@ async function generateFlowHash(
|
||||
}
|
||||
return { ...hashes, [TOP_HASH]: await generateHash(JSON.stringify(hashes)) };
|
||||
}
|
||||
/**
|
||||
* Result of generating flow locks, including which scripts were updated
|
||||
*/
|
||||
export interface FlowLocksResult {
|
||||
path: string;
|
||||
updatedScripts: string[];
|
||||
}
|
||||
|
||||
export async function generateFlowLockInternal(
|
||||
folder: string,
|
||||
dryRun: boolean,
|
||||
@@ -59,8 +69,10 @@ export async function generateFlowLockInternal(
|
||||
defaultTs?: "bun" | "deno";
|
||||
},
|
||||
justUpdateMetadataLock?: boolean,
|
||||
noStaleMessage?: boolean
|
||||
): Promise<string | void> {
|
||||
noStaleMessage?: boolean,
|
||||
legacyBehaviour?: boolean,
|
||||
tree?: DoubleLinkedDependencyTree
|
||||
): Promise<string | FlowLocksResult | void> {
|
||||
if (folder.endsWith(SEP)) {
|
||||
folder = folder.substring(0, folder.length - 1);
|
||||
}
|
||||
@@ -69,33 +81,67 @@ export async function generateFlowLockInternal(
|
||||
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
|
||||
}
|
||||
|
||||
// Always get out-of-sync workspace dependencies
|
||||
const rawWorkspaceDependencies: Record<string, string> =
|
||||
await getRawWorkspaceDependencies();
|
||||
|
||||
const flowValue = (await yamlParseFile(
|
||||
folder! + SEP + "flow.yaml"
|
||||
)) as FlowFile;
|
||||
|
||||
// Filter workspace dependencies based on inline scripts' languages and annotations
|
||||
const filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
|
||||
|
||||
let hashes = await generateFlowHash(
|
||||
filteredDeps,
|
||||
folder,
|
||||
const folderNormalized = folder.replaceAll(SEP, "/");
|
||||
const inlineScriptsForTree = extractInlineScriptsForFlows(
|
||||
structuredClone(flowValue.value.modules),
|
||||
{},
|
||||
SEP,
|
||||
opts.defaultTs
|
||||
);
|
||||
).filter(s => !s.is_lock);
|
||||
|
||||
let filteredDeps: Record<string, string> = {};
|
||||
const conf = await readLockfile();
|
||||
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
|
||||
if (!legacyBehaviour && tree) {
|
||||
if (dryRun) {
|
||||
const inlineScriptPaths: string[] = [];
|
||||
for (const script of inlineScriptsForTree) {
|
||||
let content = script.content;
|
||||
if (content.startsWith("!inline ")) {
|
||||
const filePath = folder + SEP + content.replace("!inline ", "");
|
||||
try {
|
||||
content = await readFile(filePath, "utf-8");
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path));
|
||||
const language = script.language as ScriptLanguage;
|
||||
const imports = await extractRelativeImports(content, treePath, language);
|
||||
await tree.addNode(treePath, content, language, "", imports, "inline_script", folderNormalized, folder, false);
|
||||
inlineScriptPaths.push(treePath);
|
||||
}
|
||||
|
||||
const hashes = await generateFlowHash({}, folder, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
await tree.addNode(folderNormalized, "", "bun", "", inlineScriptPaths, "flow", folderNormalized, folder, isDirectlyStale);
|
||||
return;
|
||||
}
|
||||
// Second pass: get mismatched workspace deps from tree
|
||||
filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, tree.getMismatchedWorkspaceDeps(), folder);
|
||||
} else {
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
filteredDeps = await filterWorkspaceDependenciesForFlow(flowValue.value as FlowValue, rawWorkspaceDependencies, folder);
|
||||
|
||||
const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs);
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH));
|
||||
|
||||
if (!isDirectlyStale) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return remote_path;
|
||||
}
|
||||
|
||||
if (Object.keys(filteredDeps).length > 0 && !noStaleMessage) {
|
||||
@@ -109,8 +155,25 @@ export async function generateFlowLockInternal(
|
||||
}
|
||||
|
||||
|
||||
let changedScripts: string[] = [];
|
||||
|
||||
// Build mapping from on-disk file names (hash keys like "a.py") to tree paths
|
||||
// (like "folder/a.inline_script"). The tree uses extractInlineScriptsForFlows without
|
||||
// a path assigner, so paths always have .inline_script suffix, but on-disk files
|
||||
// may not (non-dotted mode).
|
||||
const fileToTreePath = new Map<string, string>();
|
||||
for (const script of inlineScriptsForTree) {
|
||||
const c = script.content;
|
||||
if (c.startsWith("!inline ")) {
|
||||
const fileName = c.replace("!inline ", "");
|
||||
const treePath = folderNormalized + "/" + path.basename(script.path, path.extname(script.path));
|
||||
fileToTreePath.set(fileName, treePath);
|
||||
}
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
const changedScripts = [];
|
||||
const hashes = await generateFlowHash(filteredDeps, folder, opts.defaultTs);
|
||||
|
||||
//find hashes that do not correspond to previous hashes
|
||||
for (const [path, hash] of Object.entries(hashes)) {
|
||||
if (path == TOP_HASH) {
|
||||
@@ -125,30 +188,44 @@ export async function generateFlowLockInternal(
|
||||
log.info(`Recomputing locks of ${changedScripts.join(", ")} in ${folder}`);
|
||||
}
|
||||
const fileReader = async (path: string) => await readFile(folder + SEP + path, "utf-8");
|
||||
// In tree mode, use the tree's staleness info (which includes transitive dependency changes)
|
||||
// to determine which scripts need relocking, instead of only content-changed ones.
|
||||
const locksToRemove = (tree && !legacyBehaviour)
|
||||
? Object.keys(hashes).filter(k => {
|
||||
if (k === TOP_HASH) return false;
|
||||
const treePath = fileToTreePath.get(k)
|
||||
?? (folderNormalized + "/" + path.basename(k, path.extname(k)));
|
||||
return tree.isStale(treePath);
|
||||
})
|
||||
: changedScripts;
|
||||
await replaceInlineScripts(
|
||||
flowValue.value.modules,
|
||||
fileReader,
|
||||
log,
|
||||
folder + SEP!,
|
||||
SEP,
|
||||
changedScripts
|
||||
locksToRemove
|
||||
);
|
||||
if (flowValue.value.failure_module) {
|
||||
await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, changedScripts);
|
||||
await replaceInlineScripts([flowValue.value.failure_module], fileReader, log, folder + SEP!, SEP, locksToRemove);
|
||||
}
|
||||
if (flowValue.value.preprocessor_module) {
|
||||
await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, changedScripts);
|
||||
await replaceInlineScripts([flowValue.value.preprocessor_module], fileReader, log, folder + SEP!, SEP, locksToRemove);
|
||||
}
|
||||
|
||||
//removeChangedLocks
|
||||
const tempScriptRefs = tree?.getTempScriptRefs(folderNormalized);
|
||||
flowValue.value = await updateFlow(
|
||||
workspace,
|
||||
flowValue.value,
|
||||
remote_path,
|
||||
filteredDeps
|
||||
filteredDeps,
|
||||
tempScriptRefs
|
||||
);
|
||||
|
||||
const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun");
|
||||
const lockAssigner = newPathAssigner(opts.defaultTs ?? "bun", {
|
||||
skipInlineScriptSuffix: getNonDottedPaths(),
|
||||
});
|
||||
const inlineScripts = extractInlineScriptsForFlows(
|
||||
flowValue.value.modules,
|
||||
{},
|
||||
@@ -173,18 +250,36 @@ export async function generateFlowLockInternal(
|
||||
);
|
||||
}
|
||||
|
||||
hashes = await generateFlowHash(
|
||||
filteredDeps,
|
||||
// Non-legacy mode excludes workspace deps from hash (tracked via tree instead)
|
||||
const depsForHash = (tree && !legacyBehaviour) ? {} : filteredDeps;
|
||||
const finalHashes = await generateFlowHash(
|
||||
depsForHash,
|
||||
folder,
|
||||
opts.defaultTs
|
||||
);
|
||||
await clearGlobalLock(folder);
|
||||
for (const [path, hash] of Object.entries(hashes)) {
|
||||
for (const [path, hash] of Object.entries(finalHashes)) {
|
||||
await updateMetadataGlobalLock(folder, hash, path);
|
||||
}
|
||||
if (!noStaleMessage) {
|
||||
log.info(colors.green(`Flow ${remote_path} lockfiles updated`));
|
||||
}
|
||||
|
||||
// Return the list of updated scripts (extract just the filename from the path)
|
||||
// In tree mode, use the same staleness-aware list we used for lock removal
|
||||
const relocked = (tree && !legacyBehaviour)
|
||||
? Object.keys(finalHashes).filter(k => {
|
||||
if (k === TOP_HASH) return false;
|
||||
const treePath = fileToTreePath.get(k)
|
||||
?? (folderNormalized + "/" + path.basename(k, path.extname(k)));
|
||||
return tree.isStale(treePath);
|
||||
})
|
||||
: changedScripts;
|
||||
const updatedScripts = relocked.map(p => {
|
||||
const parts = p.split(SEP);
|
||||
return parts[parts.length - 1].replace(/\.[^.]+$/, ""); // Remove extension
|
||||
});
|
||||
return { path: remote_path, updatedScripts };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -218,7 +313,8 @@ export async function updateFlow(
|
||||
workspace: Workspace,
|
||||
flow_value: FlowValue,
|
||||
remotePath: string,
|
||||
rawWorkspaceDependencies: Record<string, string>
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<FlowValue | undefined> {
|
||||
let rawResponse;
|
||||
|
||||
@@ -243,6 +339,9 @@ export async function updateFlow(
|
||||
path: remotePath,
|
||||
use_local_lockfiles: true,
|
||||
raw_workspace_dependencies: rawWorkspaceDependencies,
|
||||
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? { temp_script_refs: tempScriptRefs }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
@@ -261,6 +360,9 @@ export async function updateFlow(
|
||||
body: JSON.stringify({
|
||||
flow_value,
|
||||
path: remotePath,
|
||||
...(tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? { temp_script_refs: tempScriptRefs }
|
||||
: {}),
|
||||
}),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -10,23 +10,31 @@ import * as log from "../../core/log.ts";
|
||||
import {
|
||||
generateScriptMetadataInternal,
|
||||
getRawWorkspaceDependencies,
|
||||
readLockfile,
|
||||
checkifMetadataUptodate,
|
||||
} from "../../utils/metadata.ts";
|
||||
import { generateFlowLockInternal } from "../flow/flow_metadata.ts";
|
||||
import { generateAppLocksInternal, getAppFolders } from "../app/app_metadata.ts";
|
||||
import { generateFlowLockInternal, FlowLocksResult } from "../flow/flow_metadata.ts";
|
||||
import { generateAppLocksInternal, getAppFolders, AppLocksResult } from "../app/app_metadata.ts";
|
||||
import {
|
||||
elementsToMap,
|
||||
FSFSElement,
|
||||
ignoreF,
|
||||
} from "../sync/sync.ts";
|
||||
import { exts } from "../script/script.ts";
|
||||
import { isFlowPath, isAppPath } from "../../utils/resource_folders.ts";
|
||||
import { isFolderResourcePathAnyFormat, isScriptModulePath, isModuleEntryPoint } from "../../utils/resource_folders.ts";
|
||||
import { listSyncCodebases } from "../../utils/codebase.ts";
|
||||
import {
|
||||
DoubleLinkedDependencyTree,
|
||||
uploadScripts,
|
||||
ItemType,
|
||||
} from "../../utils/dependency_tree.ts";
|
||||
|
||||
interface StaleItem {
|
||||
type: "script" | "flow" | "app";
|
||||
type: ItemType;
|
||||
path: string;
|
||||
folder: string;
|
||||
isRawApp?: boolean;
|
||||
staleReason?: string;
|
||||
}
|
||||
|
||||
async function generateMetadata(
|
||||
@@ -38,6 +46,7 @@ async function generateMetadata(
|
||||
skipScripts?: boolean;
|
||||
skipFlows?: boolean;
|
||||
skipApps?: boolean;
|
||||
strictFolderBoundaries?: boolean;
|
||||
} & SyncOptions,
|
||||
folder?: string
|
||||
) {
|
||||
@@ -49,12 +58,10 @@ async function generateMetadata(
|
||||
await requireLogin(opts);
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(false);
|
||||
const codebases = await listSyncCodebases(opts);
|
||||
const ignore = await ignoreF(opts);
|
||||
|
||||
const staleItems: StaleItem[] = [];
|
||||
|
||||
// --schema-only implies skipping flows and apps (they only have locks, no schemas)
|
||||
const skipScripts = opts.skipScripts ?? false;
|
||||
const skipFlows = opts.skipFlows ?? opts.schemaOnly ?? false;
|
||||
@@ -70,7 +77,11 @@ async function generateMetadata(
|
||||
return;
|
||||
}
|
||||
|
||||
log.info(colors.gray(`Checking ${checking.join(", ")}...`));
|
||||
log.info(`Checking ${checking.join(", ")}...`);
|
||||
|
||||
// Build dependency tree for relative import tracking
|
||||
const tree = new DoubleLinkedDependencyTree();
|
||||
tree.setWorkspaceDeps(rawWorkspaceDependencies);
|
||||
|
||||
// === Collect stale scripts ===
|
||||
if (!skipScripts) {
|
||||
@@ -81,8 +92,8 @@ async function generateMetadata(
|
||||
return (
|
||||
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
|
||||
ignore(p, isD) ||
|
||||
isFlowPath(p) ||
|
||||
isAppPath(p)
|
||||
isFolderResourcePathAnyFormat(p) ||
|
||||
(isScriptModulePath(p) && !isModuleEntryPoint(p))
|
||||
);
|
||||
},
|
||||
false,
|
||||
@@ -90,19 +101,18 @@ async function generateMetadata(
|
||||
);
|
||||
|
||||
for (const e of Object.keys(scriptElems)) {
|
||||
const candidate = await generateScriptMetadataInternal(
|
||||
await generateScriptMetadataInternal(
|
||||
e,
|
||||
workspace,
|
||||
opts,
|
||||
true, // dryRun
|
||||
true, // dryRun - populate tree
|
||||
true, // noStaleMessage
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
false
|
||||
false,
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
if (candidate) {
|
||||
staleItems.push({ type: "script", path: candidate, folder: e });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -124,18 +134,17 @@ async function generateMetadata(
|
||||
)
|
||||
).map((x) => x.substring(0, x.lastIndexOf(SEP)));
|
||||
|
||||
for (const folder of flowElems) {
|
||||
const candidate = await generateFlowLockInternal(
|
||||
folder,
|
||||
true, // dryRun
|
||||
for (const flowFolder of flowElems) {
|
||||
await generateFlowLockInternal(
|
||||
flowFolder,
|
||||
true, // dryRun - populate tree
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true // noStaleMessage
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
if (candidate) {
|
||||
staleItems.push({ type: "flow", path: candidate, folder });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -159,45 +168,134 @@ async function generateMetadata(
|
||||
const appFolders = getAppFolders(elems, "app.yaml");
|
||||
|
||||
for (const appFolder of rawAppFolders) {
|
||||
const candidate = await generateAppLocksInternal(
|
||||
await generateAppLocksInternal(
|
||||
appFolder,
|
||||
true, // rawApp
|
||||
true, // dryRun
|
||||
true, // dryRun - populate tree
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true // noStaleMessage
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
if (candidate) {
|
||||
staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: true });
|
||||
}
|
||||
}
|
||||
|
||||
for (const appFolder of appFolders) {
|
||||
const candidate = await generateAppLocksInternal(
|
||||
await generateAppLocksInternal(
|
||||
appFolder,
|
||||
false, // rawApp
|
||||
true, // dryRun
|
||||
true, // dryRun - populate tree
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true // noStaleMessage
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
if (candidate) {
|
||||
staleItems.push({ type: "app", path: candidate, folder: appFolder, isRawApp: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// === Propagate staleness through imports ===
|
||||
tree.propagateStaleness();
|
||||
|
||||
// Upload stale scripts to temp storage so the backend can resolve relative imports.
|
||||
// If this fails (e.g. backend is older and doesn't have /raw_temp endpoints),
|
||||
// degrade gracefully: locks will be generated using deployed script content only.
|
||||
try {
|
||||
await uploadScripts(tree, workspace);
|
||||
} catch (e) {
|
||||
log.warn(colors.yellow(
|
||||
`Failed to upload scripts to temp storage (backend may be too old): ${e}. ` +
|
||||
`Locks will be generated using deployed script versions only — locally modified ` +
|
||||
`relative imports may not be reflected.`
|
||||
));
|
||||
}
|
||||
|
||||
// === Populate staleItems from tree ===
|
||||
const staleItems: StaleItem[] = [];
|
||||
const seenFolders = new Set<string>();
|
||||
|
||||
for (const p of tree.allPaths()) {
|
||||
const staleReason = tree.getStaleReason(p);
|
||||
if (!staleReason) continue;
|
||||
|
||||
const itemType = tree.getItemType(p)!;
|
||||
const itemFolder = tree.getFolder(p)!;
|
||||
|
||||
if (itemType === "dependencies") {
|
||||
staleItems.push({ type: itemType, path: p, folder: itemFolder, staleReason });
|
||||
} else if (itemType === "inline_script") {
|
||||
// Inline scripts are not listed separately — their parent flow/app is stale via propagation
|
||||
continue;
|
||||
} else if (itemType === "script") {
|
||||
const originalPath = tree.getOriginalPath(p)!;
|
||||
staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, staleReason });
|
||||
} else if (!seenFolders.has(itemFolder)) {
|
||||
// Flows/Apps: one entry per folder (dedupe multiple inline scripts)
|
||||
seenFolders.add(itemFolder);
|
||||
const originalPath = tree.getOriginalPath(p)!;
|
||||
staleItems.push({ type: itemType, path: originalPath, folder: itemFolder, isRawApp: tree.getIsRawApp(p), staleReason });
|
||||
}
|
||||
}
|
||||
|
||||
// === Filter by folder if specified ===
|
||||
let filteredItems = staleItems;
|
||||
if (folder) {
|
||||
// Strip trailing separator to match deprecated flow/app handler behavior
|
||||
// (see generateFlowLockInternal line 64-66, generateAppLocksInternal line 109-110)
|
||||
if (folder.endsWith(SEP)) {
|
||||
// Normalize to forward slashes (Windows users may use backslashes)
|
||||
folder = folder.replaceAll("\\", "/");
|
||||
// Strip trailing slash to match deprecated flow/app handler behavior
|
||||
if (folder.endsWith("/")) {
|
||||
folder = folder.substring(0, folder.length - 1);
|
||||
}
|
||||
filteredItems = staleItems.filter((item) => item.folder === folder || item.folder.startsWith(folder + SEP));
|
||||
// Strip file extension if user passed a specific file path (e.g. f/test/script.ts)
|
||||
const folderNoExt = folder.replace(/\.[^/.]+$/, "");
|
||||
// Check if an item is inside the specified folder
|
||||
const isInsideFolder = (item: StaleItem) => {
|
||||
const normalizedFolder = item.folder.replaceAll("\\", "/");
|
||||
const normalizedPath = item.path.replaceAll("\\", "/");
|
||||
return normalizedFolder === folder || normalizedFolder.startsWith(folder + "/")
|
||||
|| normalizedPath === folder || normalizedPath === folderNoExt;
|
||||
};
|
||||
const isPathInFolder = (p: string) => p.startsWith(folder + "/") || p === folder || p === folderNoExt;
|
||||
// Check if a tree path or any of its transitive deps is inside the folder
|
||||
const touchesFolder = (treePath: string) => {
|
||||
if (isPathInFolder(treePath)) return true;
|
||||
let found = false;
|
||||
tree.traverseTransitive(treePath, (importPath) => {
|
||||
if (isPathInFolder(importPath)) {
|
||||
found = true;
|
||||
return true; // stop early
|
||||
}
|
||||
});
|
||||
return found;
|
||||
};
|
||||
|
||||
const isRelevant = (item: StaleItem) => {
|
||||
if (isInsideFolder(item)) return true;
|
||||
if (item.type === "dependencies") return true;
|
||||
const treePath = (item.type === "script"
|
||||
? item.path.replace(/\.[^/.]+$/, "")
|
||||
: item.folder).replaceAll("\\", "/");
|
||||
return touchesFolder(treePath);
|
||||
};
|
||||
|
||||
if (opts.strictFolderBoundaries) {
|
||||
// Strict mode: only items inside the folder
|
||||
filteredItems = staleItems.filter(isInsideFolder);
|
||||
|
||||
// Warn about stale items outside the folder that would be included by default
|
||||
const excludedStale = staleItems.filter((item) => !isInsideFolder(item) && isRelevant(item) && item.type !== "dependencies");
|
||||
for (const item of excludedStale) {
|
||||
const normalizedPath = item.path.replaceAll("\\", "/");
|
||||
log.warn(colors.yellow(
|
||||
`Warning: ${normalizedPath} depends on something inside "${folder}" but is outside it — skipped due to --strict-folder-boundaries. Next generate-metadata will not detect it as stale.`
|
||||
));
|
||||
}
|
||||
} else {
|
||||
// Default: include items inside the folder and any stale importers that transitively depend on it
|
||||
filteredItems = staleItems.filter(isRelevant);
|
||||
}
|
||||
}
|
||||
|
||||
// === Show stale items and confirm ===
|
||||
@@ -210,28 +308,24 @@ async function generateMetadata(
|
||||
const scripts = filteredItems.filter((i) => i.type === "script");
|
||||
const flows = filteredItems.filter((i) => i.type === "flow");
|
||||
const apps = filteredItems.filter((i) => i.type === "app");
|
||||
const deps = filteredItems.filter((i) => i.type === "dependencies");
|
||||
|
||||
log.info("");
|
||||
log.info(`Found ${filteredItems.length} item(s) with stale metadata:`);
|
||||
log.info(`Found ${colors.bold(String(filteredItems.length))} item(s) with stale metadata:`);
|
||||
|
||||
if (scripts.length > 0) {
|
||||
log.info(colors.gray(` Scripts (${scripts.length}):`));
|
||||
for (const item of scripts) {
|
||||
log.info(colors.yellow(` ${item.path}`));
|
||||
const printItems = (label: string, items: StaleItem[]) => {
|
||||
if (items.length === 0) return;
|
||||
log.info(` ${label} (${items.length}):`);
|
||||
for (const item of items) {
|
||||
const reason = item.staleReason ? colors.dim(colors.white(` — ${item.staleReason}`)) : "";
|
||||
log.info(` ~ ${item.path}` + reason);
|
||||
}
|
||||
}
|
||||
if (flows.length > 0) {
|
||||
log.info(colors.gray(` Flows (${flows.length}):`));
|
||||
for (const item of flows) {
|
||||
log.info(colors.yellow(` ${item.path}`));
|
||||
}
|
||||
}
|
||||
if (apps.length > 0) {
|
||||
log.info(colors.gray(` Apps (${apps.length}):`));
|
||||
for (const item of apps) {
|
||||
log.info(colors.yellow(` ${item.path}`));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
printItems("Workspace dependencies", deps);
|
||||
printItems("Scripts", scripts);
|
||||
printItems("Flows", flows);
|
||||
printItems("Apps", apps);
|
||||
|
||||
if (opts.dryRun) {
|
||||
return;
|
||||
@@ -252,61 +346,80 @@ async function generateMetadata(
|
||||
log.info("");
|
||||
|
||||
// === Process all stale items with progress counter ===
|
||||
const total = filteredItems.length;
|
||||
const mismatchedWorkspaceDeps = tree.getMismatchedWorkspaceDeps();
|
||||
const total = filteredItems.length - deps.length;
|
||||
const maxWidth = `[${total}/${total}]`.length;
|
||||
let current = 0;
|
||||
|
||||
const formatProgress = (n: number) => {
|
||||
const bracket = `[${n}/${total}]`;
|
||||
return colors.gray(bracket.padEnd(maxWidth, " "));
|
||||
return colors.dim(colors.white(`[${n}/${total}]`.padEnd(maxWidth, " ")));
|
||||
};
|
||||
|
||||
// Process scripts
|
||||
for (const item of scripts) {
|
||||
current++;
|
||||
log.info(`${formatProgress(current)} script ${colors.cyan(item.path)}`);
|
||||
log.info(`${formatProgress(current)} script ${item.path}`);
|
||||
await generateScriptMetadataInternal(
|
||||
item.folder,
|
||||
item.path, // originalPath with extension
|
||||
workspace,
|
||||
opts,
|
||||
false, // dryRun
|
||||
true, // noStaleMessage - we handle output
|
||||
rawWorkspaceDependencies,
|
||||
true, // noStaleMessage
|
||||
mismatchedWorkspaceDeps,
|
||||
codebases,
|
||||
false
|
||||
false,
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
}
|
||||
|
||||
// Process flows
|
||||
for (const item of flows) {
|
||||
current++;
|
||||
log.info(`${formatProgress(current)} flow ${colors.cyan(item.path)}`);
|
||||
await generateFlowLockInternal(
|
||||
item.folder,
|
||||
const result = await generateFlowLockInternal(
|
||||
item.folder.replaceAll("/", SEP),
|
||||
false, // dryRun
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true // noStaleMessage - we handle output
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
const flowResult = result as FlowLocksResult | undefined;
|
||||
const scriptsInfo = flowResult?.updatedScripts?.length
|
||||
? colors.dim(colors.white(`: ${flowResult.updatedScripts.join(", ")}`))
|
||||
: "";
|
||||
log.info(`${formatProgress(current)} flow ${item.path}${scriptsInfo}`);
|
||||
}
|
||||
|
||||
// Process apps
|
||||
for (const item of apps) {
|
||||
current++;
|
||||
log.info(`${formatProgress(current)} app ${colors.cyan(item.path)}`);
|
||||
await generateAppLocksInternal(
|
||||
item.folder,
|
||||
const result = await generateAppLocksInternal(
|
||||
item.folder.replaceAll("/", SEP),
|
||||
item.isRawApp!, // rawApp
|
||||
false, // dryRun
|
||||
workspace,
|
||||
opts,
|
||||
false,
|
||||
true // noStaleMessage - we handle output
|
||||
true, // noStaleMessage
|
||||
false, // legacyBehaviour
|
||||
tree
|
||||
);
|
||||
const appResult = result as AppLocksResult | undefined;
|
||||
const scriptsInfo = appResult?.updatedScripts?.length
|
||||
? colors.dim(colors.white(`: ${appResult.updatedScripts.join(", ")}`))
|
||||
: "";
|
||||
log.info(`${formatProgress(current)} app ${item.path}${scriptsInfo}`);
|
||||
}
|
||||
|
||||
// Persist all stale workspace dep hashes (not just filtered — deps are global, not folder-scoped)
|
||||
const allStaleDeps = staleItems.filter((i) => i.type === "dependencies");
|
||||
await tree.persistDepsHashes(allStaleDeps.map((d) => d.path));
|
||||
|
||||
log.info("");
|
||||
log.info(colors.green(`Done. Updated ${total} item(s).`));
|
||||
log.info(`Done. Updated ${colors.bold(String(total))} item(s).`);
|
||||
}
|
||||
|
||||
const command = new Command()
|
||||
@@ -319,6 +432,7 @@ const command = new Command()
|
||||
.option("--skip-scripts", "Skip processing scripts")
|
||||
.option("--skip-flows", "Skip processing flows")
|
||||
.option("--skip-apps", "Skip processing apps")
|
||||
.option("--strict-folder-boundaries", "Only update items inside the specified folder (requires folder argument)")
|
||||
.option(
|
||||
"-i --includes <patterns:file[]>",
|
||||
"Comma separated patterns to specify which files to include"
|
||||
|
||||
@@ -252,6 +252,16 @@ async function initAction(opts: InitOptions) {
|
||||
}
|
||||
}
|
||||
|
||||
// Read nonDottedPaths from config to specialize generated skills
|
||||
let nonDottedPaths = true; // default for new inits
|
||||
try {
|
||||
const { readConfigFile } = await import("../../core/conf.ts");
|
||||
const config = await readConfigFile();
|
||||
nonDottedPaths = config.nonDottedPaths ?? true;
|
||||
} catch {
|
||||
// If config can't be read, use default
|
||||
}
|
||||
|
||||
// Create guidance files (AGENTS.md, CLAUDE.md, and Claude skills)
|
||||
try {
|
||||
// Generate skills reference section for AGENTS.md
|
||||
@@ -290,6 +300,20 @@ async function initAction(opts: InitOptions) {
|
||||
|
||||
let skillContent = SKILL_CONTENT[skill.name];
|
||||
if (skillContent) {
|
||||
// Replace placeholders with actual suffixes based on nonDottedPaths
|
||||
if (nonDottedPaths) {
|
||||
skillContent = skillContent
|
||||
.replaceAll("{{FLOW_SUFFIX}}", "__flow")
|
||||
.replaceAll("{{APP_SUFFIX}}", "__app")
|
||||
.replaceAll("{{RAW_APP_SUFFIX}}", "__raw_app")
|
||||
.replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files should NOT include `.inline_script.` in their names (e.g. use `a.ts`, not `a.inline_script.ts`).");
|
||||
} else {
|
||||
skillContent = skillContent
|
||||
.replaceAll("{{FLOW_SUFFIX}}", ".flow")
|
||||
.replaceAll("{{APP_SUFFIX}}", ".app")
|
||||
.replaceAll("{{RAW_APP_SUFFIX}}", ".raw_app")
|
||||
.replaceAll("{{INLINE_SCRIPT_NAMING}}", "Inline script files use the `.inline_script.` naming convention (e.g. `a.inline_script.ts`).");
|
||||
}
|
||||
// Check if this skill has schemas that need to be appended
|
||||
const schemaMappings = SCHEMA_MAPPINGS[skill.name];
|
||||
if (schemaMappings && schemaMappings.length > 0) {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { Confirm } from "@cliffy/prompt/confirm";
|
||||
import { Table } from "@cliffy/table";
|
||||
import * as log from "../../core/log.ts";
|
||||
import { sep as SEP } from "node:path";
|
||||
import * as path from "node:path";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { deepEqual } from "../../utils/utils.ts";
|
||||
import * as wmill from "../../../gen/services.gen.ts";
|
||||
@@ -51,13 +52,19 @@ import fs from "node:fs";
|
||||
import { createTarBlob, type TarEntry } from "../../utils/tar.ts";
|
||||
|
||||
import { execSync } from "node:child_process";
|
||||
import { NewScript, Script } from "../../../gen/types.gen.ts";
|
||||
import { NewScript, Script, ScriptModule } from "../../../gen/types.gen.ts";
|
||||
import {
|
||||
isRawAppBackendPath as isRawAppBackendPathInternal,
|
||||
isAppInlineScriptPath as isAppInlineScriptPathInternal,
|
||||
isFlowInlineScriptPath as isFlowInlineScriptPathInternal,
|
||||
isFlowPath,
|
||||
isAppPath,
|
||||
isScriptModulePath,
|
||||
buildModuleFolderPath,
|
||||
getModuleFolderSuffix,
|
||||
isModuleEntryPoint,
|
||||
getScriptBasePathFromModulePath,
|
||||
isRawAppPath,
|
||||
} from "../../utils/resource_folders.ts";
|
||||
|
||||
export interface ScriptFile {
|
||||
@@ -123,7 +130,7 @@ async function push(opts: PushOptions, filePath: string) {
|
||||
[],
|
||||
undefined,
|
||||
opts,
|
||||
await getRawWorkspaceDependencies(),
|
||||
await getRawWorkspaceDependencies(true),
|
||||
codebases
|
||||
);
|
||||
log.info(colors.bold.underline.green(`Script ${filePath} pushed`));
|
||||
@@ -188,11 +195,17 @@ export async function handleScriptMetadata(
|
||||
codebases: SyncCodebase[],
|
||||
opts: GlobalOptions
|
||||
): Promise<boolean> {
|
||||
if (
|
||||
path.endsWith(".script.json") ||
|
||||
// Flat layout: my_script.script.yaml
|
||||
const isFlatMeta = path.endsWith(".script.json") ||
|
||||
path.endsWith(".script.yaml") ||
|
||||
path.endsWith(".script.lock")
|
||||
) {
|
||||
path.endsWith(".script.lock");
|
||||
// Folder layout: my_script__mod/script.yaml
|
||||
const isFolderMeta = !isFlatMeta && isScriptModulePath(path) && (
|
||||
path.endsWith("/script.yaml") ||
|
||||
path.endsWith("/script.json") ||
|
||||
path.endsWith("/script.lock")
|
||||
);
|
||||
if (isFlatMeta || isFolderMeta) {
|
||||
const contentPath = await findContentFile(path);
|
||||
return handleFile(
|
||||
contentPath,
|
||||
@@ -225,10 +238,13 @@ export async function handleFile(
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
codebases: SyncCodebase[]
|
||||
): Promise<boolean> {
|
||||
// Detect module entry point: e.g., my_script__mod/script.ts
|
||||
const moduleEntryPoint = isModuleEntryPoint(path);
|
||||
if (
|
||||
!isAppInlineScriptPath(path) &&
|
||||
!isFlowInlineScriptPath(path) &&
|
||||
!isRawAppBackendPath(path) &&
|
||||
(!isScriptModulePath(path) || moduleEntryPoint) &&
|
||||
exts.some((exts) => path.endsWith(exts))
|
||||
) {
|
||||
if (alreadySynced.includes(path)) {
|
||||
@@ -237,9 +253,9 @@ export async function handleFile(
|
||||
log.debug(`Processing local script ${path}`);
|
||||
|
||||
alreadySynced.push(path);
|
||||
const remotePath = path
|
||||
.substring(0, path.indexOf("."))
|
||||
.replaceAll(SEP, "/");
|
||||
const remotePath = moduleEntryPoint
|
||||
? getScriptBasePathFromModulePath(path)!.replaceAll(SEP, "/")
|
||||
: path.substring(0, path.indexOf(".")).replaceAll(SEP, "/");
|
||||
|
||||
const language = inferContentTypeFromFilePath(path, opts?.defaultTs);
|
||||
|
||||
@@ -391,6 +407,13 @@ export async function handleFile(
|
||||
typed.codebase = await codebase.getDigest(forceTar);
|
||||
}
|
||||
|
||||
// Scan for modules: folder layout (entry point inside __mod/) or flat layout
|
||||
const scriptBasePath = moduleEntryPoint
|
||||
? getScriptBasePathFromModulePath(path)!
|
||||
: path.substring(0, path.indexOf("."));
|
||||
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
|
||||
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, moduleEntryPoint);
|
||||
|
||||
const requestBodyCommon: NewScript = {
|
||||
content,
|
||||
description: typed?.description ?? "",
|
||||
@@ -409,7 +432,6 @@ export async function handleFile(
|
||||
deployment_message: message,
|
||||
restart_unless_cancelled: typed?.restart_unless_cancelled,
|
||||
visible_to_runner_only: typed?.visible_to_runner_only,
|
||||
no_main_func: typed?.no_main_func,
|
||||
has_preprocessor: typed?.has_preprocessor,
|
||||
priority: typed?.priority,
|
||||
concurrency_key: typed?.concurrency_key,
|
||||
@@ -419,6 +441,7 @@ export async function handleFile(
|
||||
timeout: typed?.timeout,
|
||||
on_behalf_of_email: typed?.on_behalf_of_email,
|
||||
envs: typed?.envs,
|
||||
modules: modules,
|
||||
};
|
||||
|
||||
// console.log(requestBodyCommon.codebase);
|
||||
@@ -449,7 +472,6 @@ export async function handleFile(
|
||||
Boolean(remote.restart_unless_cancelled) &&
|
||||
Boolean(typed.visible_to_runner_only) ==
|
||||
Boolean(remote.visible_to_runner_only) &&
|
||||
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
|
||||
Boolean(typed.has_preprocessor) ==
|
||||
Boolean(remote.has_preprocessor) &&
|
||||
typed.priority == Boolean(remote.priority) &&
|
||||
@@ -460,7 +482,8 @@ export async function handleFile(
|
||||
typed.debounce_delay_s == remote["debounce_delay_s"] &&
|
||||
typed.codebase == remote.codebase &&
|
||||
typed.on_behalf_of_email == remote.on_behalf_of_email &&
|
||||
deepEqual(typed.envs, remote.envs))
|
||||
deepEqual(typed.envs, remote.envs) &&
|
||||
deepEqual(modules ?? null, remote.modules ?? null))
|
||||
) {
|
||||
log.info(colors.green(`Script ${remotePath} is up to date`));
|
||||
return true;
|
||||
@@ -506,6 +529,135 @@ export async function handleFile(
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read module files from a __mod/ directory on disk.
|
||||
* Returns the modules record for the API, or undefined if no module folder exists.
|
||||
*/
|
||||
export async function readModulesFromDisk(
|
||||
moduleFolderPath: string,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
folderLayout: boolean = false,
|
||||
): Promise<Record<string, ScriptModule> | undefined> {
|
||||
if (!fs.existsSync(moduleFolderPath) || !fs.statSync(moduleFolderPath).isDirectory()) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const modules: Record<string, ScriptModule> = {};
|
||||
|
||||
// In folder layout mode, skip the entry point files (script.*, script.yaml, etc.)
|
||||
const isEntryPointFile = (name: string, isTopLevel: boolean) => {
|
||||
if (!folderLayout || !isTopLevel) return false;
|
||||
return (
|
||||
name.startsWith("script.") ||
|
||||
name === "script.lock" ||
|
||||
name === "script.yaml" ||
|
||||
name === "script.json"
|
||||
);
|
||||
};
|
||||
|
||||
function readDir(dirPath: string, relPrefix: string) {
|
||||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
const isTopLevel = relPrefix === "";
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
readDir(fullPath, relPath);
|
||||
} else if (entry.isFile() && !entry.name.endsWith(".lock") && !isEntryPointFile(entry.name, isTopLevel)) {
|
||||
// Skip lock files — they're handled as the `lock` field on ScriptModule
|
||||
if (exts.some((ext) => entry.name.endsWith(ext))) {
|
||||
const content = fs.readFileSync(fullPath, "utf-8");
|
||||
const language = inferContentTypeFromFilePath(entry.name, defaultTs);
|
||||
|
||||
// Check for an accompanying lock file (helper.lock)
|
||||
const baseName = entry.name.replace(/\.[^.]+$/, '');
|
||||
const lockPath = path.join(dirPath, baseName + ".lock");
|
||||
let lock: string | undefined;
|
||||
if (fs.existsSync(lockPath)) {
|
||||
lock = fs.readFileSync(lockPath, "utf-8");
|
||||
}
|
||||
|
||||
modules[relPath] = {
|
||||
content,
|
||||
language: language as ScriptModule["language"],
|
||||
lock: lock ?? undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
readDir(moduleFolderPath, "");
|
||||
|
||||
if (Object.keys(modules).length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
log.debug(`Found ${Object.keys(modules).length} module(s) in ${moduleFolderPath}`);
|
||||
return modules;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write module files to a __mod/ directory on disk during pull.
|
||||
*/
|
||||
export async function writeModulesToDisk(
|
||||
moduleFolderPath: string,
|
||||
modules: Record<string, ScriptModule>,
|
||||
defaultTs: "bun" | "deno" | undefined
|
||||
): Promise<void> {
|
||||
// Ensure the module folder exists
|
||||
fs.mkdirSync(moduleFolderPath, { recursive: true });
|
||||
|
||||
// Clean up stale module files that are no longer in the modules map
|
||||
const expectedFiles = new Set<string>();
|
||||
for (const [relPath, mod] of Object.entries(modules)) {
|
||||
expectedFiles.add(relPath);
|
||||
if (mod.lock) {
|
||||
expectedFiles.add(relPath.replace(/\.[^.]+$/, '') + ".lock");
|
||||
}
|
||||
}
|
||||
|
||||
function cleanDir(dirPath: string, relPrefix: string) {
|
||||
if (!fs.existsSync(dirPath) || !fs.statSync(dirPath).isDirectory()) return;
|
||||
const entries = fs.readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
if (entry.isDirectory()) {
|
||||
cleanDir(path.join(dirPath, entry.name), relPath);
|
||||
// Remove empty directories after cleaning
|
||||
try {
|
||||
const remaining = fs.readdirSync(path.join(dirPath, entry.name));
|
||||
if (remaining.length === 0) {
|
||||
fs.rmdirSync(path.join(dirPath, entry.name));
|
||||
}
|
||||
} catch {}
|
||||
} else if (!expectedFiles.has(relPath)) {
|
||||
fs.unlinkSync(path.join(dirPath, entry.name));
|
||||
}
|
||||
}
|
||||
}
|
||||
cleanDir(moduleFolderPath, "");
|
||||
|
||||
for (const [relPath, mod] of Object.entries(modules)) {
|
||||
const fullPath = path.join(moduleFolderPath, relPath);
|
||||
const dir = path.dirname(fullPath);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
|
||||
// Write the module content
|
||||
fs.writeFileSync(fullPath, mod.content, "utf-8");
|
||||
|
||||
// Write the lock file if present
|
||||
if (mod.lock) {
|
||||
const baseName = relPath.replace(/\.[^.]+$/, '');
|
||||
const lockPath = path.join(moduleFolderPath, baseName + ".lock");
|
||||
const lockDir = path.dirname(lockPath);
|
||||
fs.mkdirSync(lockDir, { recursive: true });
|
||||
fs.writeFileSync(lockPath, mod.lock, "utf-8");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function createScript(
|
||||
bundleContent: string | Blob | undefined,
|
||||
workspaceId: string,
|
||||
@@ -559,7 +711,12 @@ async function createScript(
|
||||
}
|
||||
|
||||
export async function findContentFile(filePath: string) {
|
||||
const candidates = filePath.endsWith("script.json")
|
||||
// Folder layout: __mod/script.yaml -> __mod/script.ts
|
||||
const isModuleFolderMeta =
|
||||
filePath.endsWith("/script.yaml") || filePath.endsWith("/script.json") || filePath.endsWith("/script.lock");
|
||||
const candidates = isModuleFolderMeta
|
||||
? exts.map((x) => filePath.replace(/\/script\.(yaml|json|lock)$/, "/script" + x))
|
||||
: filePath.endsWith("script.json")
|
||||
? exts.map((x) => filePath.replace(".script.json", x))
|
||||
: filePath.endsWith("script.lock")
|
||||
? exts.map((x) => filePath.replace(".script.lock", x))
|
||||
@@ -1004,7 +1161,7 @@ export async function generateMetadata(
|
||||
opts = await mergeConfigWithConfigFile(opts);
|
||||
const codebases = await listSyncCodebases(opts);
|
||||
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
if (scriptPath) {
|
||||
// read script metadata file
|
||||
await generateScriptMetadataInternal(
|
||||
@@ -1027,7 +1184,10 @@ export async function generateMetadata(
|
||||
(!isD && !exts.some((ext) => p.endsWith(ext))) ||
|
||||
ignore(p, isD) ||
|
||||
isFlowPath(p) ||
|
||||
isAppPath(p)
|
||||
isAppPath(p) ||
|
||||
isRawAppPath(p) ||
|
||||
// Skip module helper files; only entry points (script.{ext}) are processed
|
||||
(isScriptModulePath(p) && !isModuleEntryPoint(p))
|
||||
);
|
||||
},
|
||||
false,
|
||||
@@ -1116,6 +1276,13 @@ async function preview(
|
||||
const content = await readFile(filePath, "utf-8");
|
||||
const input = opts.data ? await resolve(opts.data) : {};
|
||||
|
||||
// Read modules from __mod/ folder if present
|
||||
const isFolderLayout = isModuleEntryPoint(filePath);
|
||||
const moduleFolderPath = isFolderLayout
|
||||
? path.dirname(filePath)
|
||||
: filePath.substring(0, filePath.indexOf(".")) + getModuleFolderSuffix();
|
||||
const modules = await readModulesFromDisk(moduleFolderPath, opts?.defaultTs, isFolderLayout);
|
||||
|
||||
// Check if this is a codebase script
|
||||
const codebase =
|
||||
language == "bun" ? findCodebase(filePath, codebases) : undefined;
|
||||
@@ -1275,6 +1442,7 @@ async function preview(
|
||||
path: filePath.substring(0, filePath.indexOf(".")).replaceAll(SEP, "/"),
|
||||
args: input,
|
||||
language: language as any,
|
||||
modules: modules ?? undefined,
|
||||
},
|
||||
});
|
||||
|
||||
|
||||
+260
-16
@@ -31,6 +31,7 @@ import {
|
||||
findResourceFile,
|
||||
handleScriptMetadata,
|
||||
removeExtensionToPath,
|
||||
filePathExtensionFromContentType,
|
||||
} from "../script/script.ts";
|
||||
|
||||
import { handleFile } from "../script/script.ts";
|
||||
@@ -68,7 +69,7 @@ import {
|
||||
readLockfile,
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
} from "../../utils/metadata.ts";
|
||||
import { OpenFlow, NativeServiceName } from "../../../gen/types.gen.ts";
|
||||
import { OpenFlow, NativeServiceName, ScriptModule } from "../../../gen/types.gen.ts";
|
||||
import { pushResource } from "../resource/resource.ts";
|
||||
import {
|
||||
newPathAssigner,
|
||||
@@ -97,6 +98,10 @@ import {
|
||||
getFolderSuffix,
|
||||
getFolderSuffixWithSep,
|
||||
getNonDottedPaths,
|
||||
isScriptModulePath,
|
||||
getModuleFolderSuffix,
|
||||
isModuleEntryPoint,
|
||||
getScriptBasePathFromModulePath,
|
||||
} from "../../utils/resource_folders.ts";
|
||||
|
||||
// Merge CLI options with effective settings, preserving CLI flags as overrides
|
||||
@@ -534,6 +539,29 @@ function ZipFSElement(
|
||||
resourceTypeToIsFileset: Record<string, boolean>,
|
||||
ignoreCodebaseChanges: boolean,
|
||||
): DynFSElement {
|
||||
// Pre-scan: find zip base paths of scripts that have modules.
|
||||
// These scripts use the folder layout: {basePath}__mod/script.{ext}
|
||||
let _moduleScriptPaths: Set<string> | null = null;
|
||||
async function getModuleScriptPaths(): Promise<Set<string>> {
|
||||
if (_moduleScriptPaths === null) {
|
||||
_moduleScriptPaths = new Set();
|
||||
for (const filename in zip.files) {
|
||||
if (filename.endsWith(".script.json") && !zip.files[filename].dir) {
|
||||
try {
|
||||
const content = await zip.files[filename].async("text");
|
||||
const parsed = JSON.parse(content);
|
||||
if (parsed.modules && Object.keys(parsed.modules).length > 0) {
|
||||
_moduleScriptPaths.add(
|
||||
filename.slice(0, -".script.json".length)
|
||||
);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
return _moduleScriptPaths;
|
||||
}
|
||||
|
||||
async function _internal_file(
|
||||
p: string,
|
||||
f: JSZip.JSZipObject,
|
||||
@@ -575,7 +603,22 @@ function ZipFSElement(
|
||||
}
|
||||
}
|
||||
|
||||
const finalPath = transformPath();
|
||||
let finalPath = transformPath();
|
||||
|
||||
// Redirect content files for scripts with modules into __mod/ folder
|
||||
if (kind == "other" && exts.some((ext) => p.endsWith(ext))) {
|
||||
const normalizedP = p.replace(/^\.[\\/]/, "");
|
||||
const moduleScripts = await getModuleScriptPaths();
|
||||
for (const basePath of moduleScripts) {
|
||||
if (normalizedP.startsWith(basePath + ".")) {
|
||||
const ext = normalizedP.slice(basePath.length); // e.g., ".ts", ".py"
|
||||
const dir = path.dirname(finalPath);
|
||||
const base = path.basename(basePath);
|
||||
finalPath = path.join(dir, base + getModuleFolderSuffix(), "script" + ext);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const r = [
|
||||
{
|
||||
@@ -893,15 +936,23 @@ function ZipFSElement(
|
||||
log.error(`Failed to parse script.yaml at path: ${p}`);
|
||||
throw error;
|
||||
}
|
||||
const hasModules = parsed["modules"] && Object.keys(parsed["modules"]).length > 0;
|
||||
if (
|
||||
parsed["lock"] &&
|
||||
parsed["lock"] != "" &&
|
||||
parsed["codebase"] == undefined
|
||||
) {
|
||||
parsed["lock"] =
|
||||
"!inline " +
|
||||
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
|
||||
".lock";
|
||||
if (hasModules) {
|
||||
// Lock lives inside __mod/ folder as script.lock
|
||||
const scriptBase = removeSuffix(removeSuffix(p.replaceAll(SEP, "/"), ".json"), ".script");
|
||||
parsed["lock"] =
|
||||
"!inline " + scriptBase + getModuleFolderSuffix() + "/script.lock";
|
||||
} else {
|
||||
parsed["lock"] =
|
||||
"!inline " +
|
||||
removeSuffix(p.replaceAll(SEP, "/"), ".json") +
|
||||
".lock";
|
||||
}
|
||||
} else if (parsed["lock"] == "") {
|
||||
parsed["lock"] = "";
|
||||
} else {
|
||||
@@ -910,6 +961,8 @@ function ZipFSElement(
|
||||
if (ignoreCodebaseChanges && parsed["codebase"]) {
|
||||
parsed["codebase"] = undefined;
|
||||
}
|
||||
// Modules are stored as files in __mod/ folder, not in metadata
|
||||
delete parsed["modules"];
|
||||
return useYaml
|
||||
? yamlStringify(parsed, yamlOptions)
|
||||
: JSON.stringify(parsed, null, 2);
|
||||
@@ -969,16 +1022,71 @@ function ZipFSElement(
|
||||
throw error;
|
||||
}
|
||||
const lock = parsed["lock"];
|
||||
const scriptModules: Record<string, ScriptModule> | undefined = parsed["modules"];
|
||||
const hasModules = scriptModules && Object.keys(scriptModules).length > 0;
|
||||
|
||||
// Compute base path and module folder
|
||||
const metaExt = useYaml ? ".yaml" : ".json";
|
||||
const scriptBasePath = removeSuffix(
|
||||
removeSuffix(finalPath, metaExt),
|
||||
".script"
|
||||
);
|
||||
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
|
||||
|
||||
if (hasModules) {
|
||||
// Redirect metadata into __mod/script.yaml
|
||||
r[0].path = path.join(moduleFolderPath, "script" + metaExt);
|
||||
}
|
||||
|
||||
if (lock && lock != "") {
|
||||
r.push({
|
||||
isDirectory: false,
|
||||
path: removeSuffix(finalPath, ".json") + ".lock",
|
||||
path: hasModules
|
||||
? path.join(moduleFolderPath, "script.lock")
|
||||
: removeSuffix(finalPath, metaExt) + ".lock",
|
||||
async *getChildren() {},
|
||||
async getContentText() {
|
||||
return lock;
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Extract script modules into __mod/ folder
|
||||
if (hasModules) {
|
||||
r.push({
|
||||
isDirectory: true,
|
||||
path: moduleFolderPath,
|
||||
async *getChildren() {
|
||||
for (const [relPath, mod] of Object.entries(scriptModules!)) {
|
||||
// Yield the module content file
|
||||
yield {
|
||||
isDirectory: false,
|
||||
path: path.join(moduleFolderPath, relPath),
|
||||
async *getChildren() {},
|
||||
async getContentText() {
|
||||
return mod.content;
|
||||
},
|
||||
};
|
||||
|
||||
// Yield the module lock file if present
|
||||
if (mod.lock) {
|
||||
const baseName = relPath.replace(/\.[^.]+$/, '');
|
||||
yield {
|
||||
isDirectory: false,
|
||||
path: path.join(moduleFolderPath, baseName + ".lock"),
|
||||
async *getChildren() {},
|
||||
async getContentText() {
|
||||
return mod.lock!;
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
},
|
||||
async getContentText() {
|
||||
throw new Error("Cannot get content of directory");
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
if (kind == "resource") {
|
||||
const content = await f.async("text");
|
||||
@@ -1154,6 +1262,12 @@ export async function elementsToMap(
|
||||
continue;
|
||||
}
|
||||
const path = entry.path;
|
||||
// Include module files in the map so they're compared for changes,
|
||||
// but they're pushed as part of their parent script via handleFile
|
||||
if (isScriptModulePath(path)) {
|
||||
map[path] = await entry.getContentText();
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
!isFileResource(path) &&
|
||||
!isFilesetResource(path) &&
|
||||
@@ -1600,6 +1714,11 @@ const isNotWmillFile = (p: string, isDirectory: boolean) => {
|
||||
);
|
||||
}
|
||||
|
||||
// Files inside __mod/ folders are script module files — always valid wmill files
|
||||
if (isScriptModulePath(p)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
const typ = getTypeStrFromPath(p);
|
||||
if (
|
||||
@@ -1744,6 +1863,37 @@ async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
|
||||
if (!tracker.rawApps.includes(folder)) {
|
||||
tracker.rawApps.push(folder);
|
||||
}
|
||||
} else if (isScriptModulePath(p)) {
|
||||
if (isModuleEntryPoint(p)) {
|
||||
// Entry point (e.g. __mod/script.ts) IS the parent script content file
|
||||
if (!tracker.scripts.includes(p)) {
|
||||
tracker.scripts.push(p);
|
||||
}
|
||||
} else {
|
||||
// Module file changed — find the parent script content file
|
||||
const moduleSuffix = getModuleFolderSuffix() + "/";
|
||||
const idx = p.indexOf(moduleSuffix);
|
||||
if (idx !== -1) {
|
||||
const scriptBasePath = p.substring(0, idx);
|
||||
// Try folder layout first: __mod/script.{ext}
|
||||
try {
|
||||
const contentPath = await findContentFile(scriptBasePath + getModuleFolderSuffix() + "/script.yaml");
|
||||
if (contentPath && !tracker.scripts.includes(contentPath)) {
|
||||
tracker.scripts.push(contentPath);
|
||||
}
|
||||
} catch {
|
||||
// Fall back to flat layout: scriptBasePath.script.yaml
|
||||
try {
|
||||
const contentPath = await findContentFile(scriptBasePath + ".script.yaml");
|
||||
if (contentPath && !tracker.scripts.includes(contentPath)) {
|
||||
tracker.scripts.push(contentPath);
|
||||
}
|
||||
} catch {
|
||||
// ignore — content file not found
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (!tracker.scripts.includes(p)) {
|
||||
tracker.scripts.push(p);
|
||||
@@ -1776,6 +1926,61 @@ async function buildTracker(changes: Change[]) {
|
||||
return tracker;
|
||||
}
|
||||
|
||||
/**
|
||||
* When a module file changes, find and push the parent script.
|
||||
* The parent script's handleFile will read the __mod/ folder and include all modules.
|
||||
*/
|
||||
async function pushParentScriptForModule(
|
||||
modulePath: string,
|
||||
workspace: Workspace,
|
||||
alreadySynced: string[],
|
||||
message: string | undefined,
|
||||
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
codebases: SyncCodebase[],
|
||||
): Promise<void> {
|
||||
const moduleSuffix = getModuleFolderSuffix() + "/";
|
||||
const idx = modulePath.indexOf(moduleSuffix);
|
||||
if (idx === -1) return;
|
||||
const scriptBasePath = modulePath.substring(0, idx);
|
||||
const moduleFolderPath = scriptBasePath + getModuleFolderSuffix();
|
||||
|
||||
// Try folder layout first: look for script.{ext} inside __mod/
|
||||
try {
|
||||
const entryPoint = await findContentFile(moduleFolderPath + "/script.yaml");
|
||||
if (entryPoint) {
|
||||
await handleFile(
|
||||
entryPoint,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
return;
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Fall back to flat layout: look for content file alongside __mod/
|
||||
try {
|
||||
const contentPath = await findContentFile(scriptBasePath + ".script.yaml");
|
||||
if (contentPath) {
|
||||
await handleFile(
|
||||
contentPath,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
}
|
||||
} catch {
|
||||
log.debug(`Could not find parent script for module: ${modulePath}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function pull(
|
||||
opts: GlobalOptions &
|
||||
SyncOptions & { repository?: string; promotion?: string; branch?: string },
|
||||
@@ -2075,7 +2280,7 @@ export async function pull(
|
||||
|
||||
const tracker: ChangeTracker = await buildTracker(changes);
|
||||
const rawWorkspaceDependencies: Record<string, string> =
|
||||
await getRawWorkspaceDependencies();
|
||||
await getRawWorkspaceDependencies(true);
|
||||
|
||||
for (const change of tracker.scripts) {
|
||||
await generateScriptMetadataInternal(
|
||||
@@ -2406,7 +2611,7 @@ export async function push(
|
||||
false, // els1 (local) is not the remote source
|
||||
);
|
||||
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies();
|
||||
const rawWorkspaceDependencies = await getRawWorkspaceDependencies(true);
|
||||
|
||||
const tracker: ChangeTracker = await buildTracker(changes);
|
||||
|
||||
@@ -2452,7 +2657,7 @@ export async function push(
|
||||
true,
|
||||
);
|
||||
if (stale) {
|
||||
staleFlows.push(stale);
|
||||
staleFlows.push(stale as string);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2477,7 +2682,7 @@ export async function push(
|
||||
true,
|
||||
);
|
||||
if (stale) {
|
||||
staleApps.push(stale);
|
||||
staleApps.push(stale as string);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2492,7 +2697,7 @@ export async function push(
|
||||
true,
|
||||
);
|
||||
if (stale) {
|
||||
staleApps.push(stale);
|
||||
staleApps.push(stale as string);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2704,6 +2909,21 @@ export async function push(
|
||||
await writeFile(stateTarget, change.after, "utf-8");
|
||||
}
|
||||
continue;
|
||||
} else if (isScriptModulePath(change.path)) {
|
||||
// Module file changed — push the parent script
|
||||
await pushParentScriptForModule(
|
||||
change.path,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
opts.message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
if (stateTarget) {
|
||||
await writeFile(stateTarget, change.after, "utf-8");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (stateTarget) {
|
||||
await mkdir(path.dirname(stateTarget), { recursive: true });
|
||||
@@ -2828,6 +3048,17 @@ export async function push(
|
||||
)
|
||||
) {
|
||||
continue;
|
||||
} else if (isScriptModulePath(change.path)) {
|
||||
await pushParentScriptForModule(
|
||||
change.path,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
opts.message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (stateTarget) {
|
||||
await mkdir(path.dirname(stateTarget), { recursive: true });
|
||||
@@ -2869,6 +3100,19 @@ export async function push(
|
||||
if (change.path.endsWith(".lock")) {
|
||||
continue;
|
||||
}
|
||||
if (isScriptModulePath(change.path)) {
|
||||
// Module file deleted — push the parent script (which will now have fewer modules)
|
||||
await pushParentScriptForModule(
|
||||
change.path,
|
||||
workspace,
|
||||
alreadySynced,
|
||||
opts.message,
|
||||
opts,
|
||||
rawWorkspaceDependencies,
|
||||
codebases,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
const typ = getTypeStrFromPath(change.path);
|
||||
|
||||
if (typ == "script") {
|
||||
@@ -3244,8 +3488,8 @@ const command = new Command()
|
||||
"Use promotionOverrides from the specified branch instead of regular overrides",
|
||||
)
|
||||
.option(
|
||||
"--branch <branch:string>",
|
||||
"Override the current git branch (works even outside a git repository)",
|
||||
"--branch, --env <branch:string>",
|
||||
"Override the current git branch/environment (works even outside a git repository)",
|
||||
)
|
||||
.action(pull as any)
|
||||
.command("push")
|
||||
@@ -3300,8 +3544,8 @@ const command = new Command()
|
||||
"Specify repository path (e.g., u/user/repo) when multiple repositories exist",
|
||||
)
|
||||
.option(
|
||||
"--branch <branch:string>",
|
||||
"Override the current git branch (works even outside a git repository)",
|
||||
"--branch, --env <branch:string>",
|
||||
"Override the current git branch/environment (works even outside a git repository)",
|
||||
)
|
||||
.option("--lint", "Run lint validation before pushing")
|
||||
.option(
|
||||
|
||||
@@ -568,11 +568,11 @@ const command = new Command()
|
||||
.action(listRemote as any)
|
||||
.command("bind")
|
||||
.description("Bind the current Git branch to the active workspace")
|
||||
.option("--branch <branch:string>", "Specify branch (defaults to current)")
|
||||
.option("--branch, --env <branch:string>", "Specify branch/environment (defaults to current)")
|
||||
.action((opts) => bind(opts as any, true))
|
||||
.command("unbind")
|
||||
.description("Remove workspace binding from the current Git branch")
|
||||
.option("--branch <branch:string>", "Specify branch (defaults to current)")
|
||||
.option("--branch, --env <branch:string>", "Specify branch/environment (defaults to current)")
|
||||
.action((opts) => bind(opts as any, false))
|
||||
.command("fork")
|
||||
.description("Create a forked workspace")
|
||||
|
||||
@@ -75,6 +75,8 @@ export interface SyncOptions {
|
||||
};
|
||||
};
|
||||
};
|
||||
// Alias for gitBranches - for users who prefer environment-based terminology
|
||||
environments?: SyncOptions["gitBranches"];
|
||||
// Legacy field - deprecated, use gitBranches instead
|
||||
git_branches?: {
|
||||
commonSpecificItems?: {
|
||||
@@ -231,6 +233,18 @@ export async function readConfigFile(): Promise<SyncOptions> {
|
||||
}
|
||||
}
|
||||
|
||||
// Handle environments -> gitBranches alias (permanent alias, not a deprecation)
|
||||
if (conf && "environments" in conf) {
|
||||
if (!conf.gitBranches) {
|
||||
conf.gitBranches = conf.environments as any;
|
||||
} else {
|
||||
log.warn(
|
||||
"⚠️ Both 'environments' and 'gitBranches' found in wmill.yaml. Using 'gitBranches' and ignoring 'environments'."
|
||||
);
|
||||
}
|
||||
delete (conf as any).environments;
|
||||
}
|
||||
|
||||
// Handle git_branches to gitBranches migration
|
||||
if (conf && "git_branches" in conf) {
|
||||
if (!conf.gitBranches) {
|
||||
|
||||
@@ -53,6 +53,7 @@ export interface SimplifiedSettings {
|
||||
mute_critical_alerts?: boolean;
|
||||
color?: string;
|
||||
operator_settings?: any;
|
||||
datatable?: any;
|
||||
slack_team_id?: string;
|
||||
slack_name?: string;
|
||||
slack_command_script?: string;
|
||||
@@ -100,6 +101,7 @@ export function migrateToGroupedFormat(settings: any): SimplifiedSettings {
|
||||
if (settings.mute_critical_alerts !== undefined) result.mute_critical_alerts = settings.mute_critical_alerts;
|
||||
if (settings.color !== undefined) result.color = settings.color;
|
||||
if (settings.operator_settings !== undefined) result.operator_settings = settings.operator_settings;
|
||||
if (settings.datatable !== undefined) result.datatable = settings.datatable;
|
||||
if (settings.slack_team_id !== undefined) result.slack_team_id = settings.slack_team_id;
|
||||
if (settings.slack_name !== undefined) result.slack_name = settings.slack_name;
|
||||
if (settings.slack_command_script !== undefined) result.slack_command_script = settings.slack_command_script;
|
||||
@@ -192,6 +194,7 @@ export async function pushWorkspaceSettings(
|
||||
mute_critical_alerts: remoteSettings.mute_critical_alerts,
|
||||
color: remoteSettings.color,
|
||||
operator_settings: remoteSettings.operator_settings,
|
||||
datatable: remoteSettings.datatable,
|
||||
slack_team_id: remoteSettings.slack_team_id,
|
||||
slack_name: remoteSettings.slack_name,
|
||||
slack_command_script: remoteSettings.slack_command_script,
|
||||
@@ -382,6 +385,14 @@ export async function pushWorkspaceSettings(
|
||||
});
|
||||
}
|
||||
|
||||
if (!deepEqual(localSettings.datatable, settings.datatable)) {
|
||||
log.debug(`Updating datatable config...`);
|
||||
await wmill.editDataTableConfig({
|
||||
workspace,
|
||||
requestBody: { settings: localSettings.datatable ?? { datatables: {} } },
|
||||
});
|
||||
}
|
||||
|
||||
if (localSettings.slack_command_script != settings.slack_command_script) {
|
||||
log.debug(`Updating slack command script...`);
|
||||
await wmill.editSlackCommand({
|
||||
|
||||
+74
-11
File diff suppressed because one or more lines are too long
+1
-1
@@ -68,7 +68,7 @@ export {
|
||||
workspaceAdd,
|
||||
};
|
||||
|
||||
export const VERSION = "1.655.0";
|
||||
export const VERSION = "1.662.0";
|
||||
|
||||
// Re-exported from constants.ts to maintain backwards compatibility
|
||||
export { WM_FORK_PREFIX } from "./core/constants.ts";
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
isRawAppPath,
|
||||
extractResourceName,
|
||||
buildFolderPath,
|
||||
isScriptModulePath,
|
||||
} from "./utils/resource_folders.ts";
|
||||
|
||||
export interface DifferenceCreate {
|
||||
@@ -259,6 +260,9 @@ export function getTypeStrFromPath(
|
||||
| "settings"
|
||||
| "encryption_key"
|
||||
| "workspace_dependencies" {
|
||||
if (isScriptModulePath(p)) {
|
||||
return "script";
|
||||
}
|
||||
if (isFlowPath(p)) {
|
||||
return "flow";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,373 @@
|
||||
/**
|
||||
* Double-linked dependency tree for tracking script imports and propagating staleness.
|
||||
*/
|
||||
|
||||
import { Workspace } from "../commands/workspace/workspace.ts";
|
||||
import * as wmill from "../../gen/services.gen.ts";
|
||||
import type { ScriptLang } from "../../gen/types.gen.ts";
|
||||
import { ScriptLanguage } from "./script_common.ts";
|
||||
import {
|
||||
filterWorkspaceDependencies,
|
||||
generateScriptHash,
|
||||
checkifMetadataUptodate,
|
||||
workspaceDependenciesPathToLanguageAndFilename,
|
||||
updateMetadataGlobalLock,
|
||||
} from "./metadata.ts";
|
||||
import { generateHash } from "./utils.ts";
|
||||
|
||||
/**
|
||||
* Diff local scripts against deployed versions, upload only those that differ.
|
||||
* Only uploaded (mismatched) scripts get contentHash set, so flatten() returns
|
||||
* temp_script_refs only for scripts the backend can't resolve from deployed versions.
|
||||
*/
|
||||
export async function uploadScripts(
|
||||
tree: DoubleLinkedDependencyTree,
|
||||
workspace: Workspace
|
||||
): Promise<void> {
|
||||
// Split into scripts vs workspace deps and compute SHA256(content) for each
|
||||
const scriptHashes: Record<string, string> = {};
|
||||
const workspaceDeps: { path: string; language: ScriptLang; name?: string; hash: string }[] = [];
|
||||
|
||||
for (const path of tree.allPaths()) {
|
||||
const content = tree.getContent(path);
|
||||
const itemType = tree.getItemType(path);
|
||||
|
||||
if (itemType === "dependencies") {
|
||||
// Empty string is valid for workspace deps (means "no deps") — only skip undefined
|
||||
if (content === undefined) continue;
|
||||
const info = workspaceDependenciesPathToLanguageAndFilename(path);
|
||||
if (info) {
|
||||
const hash = await generateHash(content);
|
||||
workspaceDeps.push({ path, language: info.language as ScriptLang, name: info.name, hash });
|
||||
}
|
||||
} else if (itemType === "script") {
|
||||
if (!content) continue;
|
||||
const hash = await generateHash(content);
|
||||
scriptHashes[path] = hash;
|
||||
}
|
||||
// Skip inline_script, flow, app — they don't need temp storage uploads
|
||||
}
|
||||
|
||||
if (Object.keys(scriptHashes).length === 0 && workspaceDeps.length === 0) return;
|
||||
|
||||
// Single batch query: find which scripts/deps differ from deployed versions
|
||||
const mismatched = await wmill.diffRawScriptsWithDeployed({
|
||||
workspace: workspace.workspaceId,
|
||||
requestBody: {
|
||||
scripts: scriptHashes,
|
||||
workspace_deps: workspaceDeps,
|
||||
},
|
||||
});
|
||||
|
||||
// Upload only mismatched scripts to temp storage
|
||||
for (const path of mismatched) {
|
||||
const content = tree.getContent(path);
|
||||
const itemType = tree.getItemType(path);
|
||||
|
||||
if (itemType === "dependencies") {
|
||||
// Workspace deps don't need temp storage — just mark as mismatched.
|
||||
// Empty string is valid (means the dep file was emptied locally).
|
||||
if (content !== undefined) {
|
||||
tree.setContentHash(path, "mismatched");
|
||||
}
|
||||
} else if (content) {
|
||||
const hash = await wmill.storeRawScriptTemp({
|
||||
workspace: workspace.workspaceId,
|
||||
requestBody: content,
|
||||
});
|
||||
tree.setContentHash(path, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export type ItemType = "script" | "inline_script" | "flow" | "app" | "dependencies";
|
||||
|
||||
interface DependencyNode {
|
||||
content: string;
|
||||
stalenessHash: string; // Hash for staleness detection (includes deps, content, metadata)
|
||||
contentHash?: string; // Hash for temp storage lookup (content only)
|
||||
language: ScriptLanguage;
|
||||
metadata: string;
|
||||
imports: Set<string>;
|
||||
importedBy: Set<string>;
|
||||
staleReason?: string;
|
||||
// Item metadata for generate-metadata command
|
||||
itemType: ItemType;
|
||||
folder: string; // Folder path (for flows/apps) or remote path (for scripts)
|
||||
originalPath: string; // Original path passed to handler (with extension for scripts)
|
||||
isRawApp?: boolean; // Only set for apps
|
||||
isDirectlyStale: boolean; // True if this item's content changed (vs transitively stale)
|
||||
}
|
||||
|
||||
export class DoubleLinkedDependencyTree {
|
||||
private nodes: Map<string, DependencyNode> = new Map();
|
||||
private workspaceDeps: Record<string, string> = {};
|
||||
|
||||
setWorkspaceDeps(deps: Record<string, string>): void {
|
||||
this.workspaceDeps = deps;
|
||||
}
|
||||
|
||||
async addNode(
|
||||
path: string,
|
||||
content: string,
|
||||
language: ScriptLanguage,
|
||||
metadata: string,
|
||||
imports: string[],
|
||||
itemType: ItemType,
|
||||
folder: string,
|
||||
originalPath: string,
|
||||
isDirectlyStale: boolean,
|
||||
isRawApp?: boolean
|
||||
): Promise<void> {
|
||||
const hasWorkspaceDeps = itemType === "script" || itemType === "inline_script";
|
||||
const filteredDeps = hasWorkspaceDeps
|
||||
? filterWorkspaceDependencies(this.workspaceDeps, content, language)
|
||||
: {};
|
||||
const stalenessHash = await generateScriptHash({}, content, metadata);
|
||||
|
||||
if (!this.nodes.has(path)) {
|
||||
this.nodes.set(path, {
|
||||
content: "", stalenessHash: "", language: "deno", metadata: "",
|
||||
imports: new Set(), importedBy: new Set(),
|
||||
itemType: "script", folder: "", originalPath: "", isDirectlyStale: false,
|
||||
});
|
||||
}
|
||||
const node = this.nodes.get(path)!;
|
||||
node.content = content;
|
||||
node.stalenessHash = stalenessHash;
|
||||
node.language = language;
|
||||
node.metadata = metadata;
|
||||
node.itemType = itemType;
|
||||
node.folder = folder;
|
||||
node.originalPath = originalPath;
|
||||
node.isDirectlyStale = isDirectlyStale;
|
||||
node.isRawApp = isRawApp;
|
||||
|
||||
// Create nodes for referenced workspace deps with content and language.
|
||||
const filteredDepsPaths = Object.keys(filteredDeps);
|
||||
for (const depsPath of filteredDepsPaths) {
|
||||
if (!this.nodes.has(depsPath)) {
|
||||
const depsInfo = workspaceDependenciesPathToLanguageAndFilename(depsPath);
|
||||
const contentHash = await generateHash(filteredDeps[depsPath] + depsPath);
|
||||
const isUpToDate = await checkifMetadataUptodate(depsPath, contentHash, undefined);
|
||||
this.nodes.set(depsPath, {
|
||||
content: filteredDeps[depsPath],
|
||||
stalenessHash: "", language: depsInfo?.language ?? "deno", metadata: "",
|
||||
imports: new Set(), importedBy: new Set(),
|
||||
itemType: "dependencies", folder: "", originalPath: depsPath,
|
||||
isDirectlyStale: !isUpToDate,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const allImports = [...imports, ...filteredDepsPaths];
|
||||
for (const importPath of allImports) {
|
||||
node.imports.add(importPath);
|
||||
|
||||
if (!this.nodes.has(importPath)) {
|
||||
this.nodes.set(importPath, {
|
||||
content: "", stalenessHash: "", language: "deno", metadata: "",
|
||||
imports: new Set(), importedBy: new Set(),
|
||||
itemType: "script", folder: "", originalPath: "", isDirectlyStale: false,
|
||||
});
|
||||
}
|
||||
this.nodes.get(importPath)!.importedBy.add(path);
|
||||
}
|
||||
}
|
||||
|
||||
getContent(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.content;
|
||||
}
|
||||
|
||||
getStalenessHash(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.stalenessHash;
|
||||
}
|
||||
|
||||
getContentHash(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.contentHash;
|
||||
}
|
||||
|
||||
setContentHash(path: string, hash: string): void {
|
||||
const node = this.nodes.get(path);
|
||||
if (node) {
|
||||
node.contentHash = hash;
|
||||
}
|
||||
}
|
||||
|
||||
getLanguage(path: string): ScriptLanguage | undefined {
|
||||
return this.nodes.get(path)?.language;
|
||||
}
|
||||
|
||||
getMetadata(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.metadata;
|
||||
}
|
||||
|
||||
getStaleReason(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.staleReason;
|
||||
}
|
||||
|
||||
getItemType(path: string): ItemType | undefined {
|
||||
return this.nodes.get(path)?.itemType;
|
||||
}
|
||||
|
||||
getFolder(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.folder;
|
||||
}
|
||||
|
||||
getIsRawApp(path: string): boolean | undefined {
|
||||
return this.nodes.get(path)?.isRawApp;
|
||||
}
|
||||
|
||||
getIsDirectlyStale(path: string): boolean {
|
||||
return this.nodes.get(path)?.isDirectlyStale ?? false;
|
||||
}
|
||||
|
||||
getOriginalPath(path: string): string | undefined {
|
||||
return this.nodes.get(path)?.originalPath;
|
||||
}
|
||||
|
||||
getImports(path: string): Set<string> | undefined {
|
||||
return this.nodes.get(path)?.imports;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true if this node has been marked stale (directly or transitively).
|
||||
*/
|
||||
isStale(path: string): boolean {
|
||||
return this.nodes.get(path)?.staleReason !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mutates the tree by removing all nodes that are not stale.
|
||||
* Uses BFS on reverse graph (importedBy) to find all stale scripts.
|
||||
* Starts from nodes with isDirectlyStale=true.
|
||||
*/
|
||||
propagateStaleness(): void {
|
||||
// Collect directly stale nodes
|
||||
const directlyStale = new Set<string>();
|
||||
for (const [path, node] of this.nodes.entries()) {
|
||||
if (node.isDirectlyStale) {
|
||||
directlyStale.add(path);
|
||||
node.staleReason = "content changed";
|
||||
}
|
||||
}
|
||||
|
||||
const allStale = new Set(directlyStale);
|
||||
const queue = [...directlyStale];
|
||||
const visited = new Set<string>();
|
||||
|
||||
while (queue.length > 0) {
|
||||
const scriptPath = queue.shift()!;
|
||||
if (visited.has(scriptPath)) continue;
|
||||
visited.add(scriptPath);
|
||||
|
||||
const node = this.nodes.get(scriptPath);
|
||||
if (!node) continue;
|
||||
|
||||
for (const importer of node.importedBy) {
|
||||
if (!allStale.has(importer)) {
|
||||
allStale.add(importer);
|
||||
queue.push(importer);
|
||||
// Set reason for transitively stale scripts
|
||||
const importerNode = this.nodes.get(importer);
|
||||
if (importerNode) importerNode.staleReason = `depends on ${scriptPath}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks all transitive imports for a node, calling the callback for each.
|
||||
* Callback may return true to stop traversing that branch.
|
||||
*/
|
||||
traverseTransitive(scriptPath: string, callback: (importPath: string, node: DependencyNode) => boolean | void): void {
|
||||
const queue = [scriptPath];
|
||||
const visited = new Set<string>();
|
||||
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift()!;
|
||||
if (visited.has(current)) continue;
|
||||
visited.add(current);
|
||||
|
||||
const node = this.nodes.get(current);
|
||||
if (!node) continue;
|
||||
|
||||
for (const importPath of node.imports) {
|
||||
const importNode = this.nodes.get(importPath);
|
||||
if (importNode) {
|
||||
const stop = callback(importPath, importNode);
|
||||
if (!stop) {
|
||||
queue.push(importPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
allPaths(): IterableIterator<string> {
|
||||
return this.nodes.keys();
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns paths of all stale nodes (those with a staleReason).
|
||||
*/
|
||||
*stalePaths(): IterableIterator<string> {
|
||||
for (const [path, node] of this.nodes.entries()) {
|
||||
if (node.staleReason) {
|
||||
yield path;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
has(path: string): boolean {
|
||||
return this.nodes.has(path);
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns workspace deps that were uploaded as mismatched with remote.
|
||||
* These need to be passed as raw_workspace_dependencies in job args
|
||||
* so the backend uses local content instead of deployed.
|
||||
*/
|
||||
getMismatchedWorkspaceDeps(): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
for (const [path, node] of this.nodes.entries()) {
|
||||
if (node.itemType === "dependencies" && node.contentHash && node.content !== undefined) {
|
||||
result[path] = node.content;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns path → contentHash for all transitive imports that have been uploaded.
|
||||
* Must be called after uploadScripts() has populated contentHash values.
|
||||
*/
|
||||
getTempScriptRefs(scriptPath: string): Record<string, string> {
|
||||
const result: Record<string, string> = {};
|
||||
this.traverseTransitive(scriptPath, (_path, node) => {
|
||||
if (node.contentHash) {
|
||||
result[_path] = node.contentHash;
|
||||
}
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Persist workspace dep hashes to wmill-lock.yaml so getRawWorkspaceDependencies
|
||||
* considers them up-to-date on the next run.
|
||||
*/
|
||||
async persistDepsHashes(depsPaths: string[]): Promise<void> {
|
||||
for (const path of depsPaths) {
|
||||
const node = this.nodes.get(path);
|
||||
if (node?.itemType === "dependencies" && node.content !== undefined) {
|
||||
const hash = await generateHash(node.content + path);
|
||||
await updateMetadataGlobalLock(path, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
get size(): number {
|
||||
return this.nodes.size;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFile, stat } from "node:fs/promises";
|
||||
import type { SyncCodebase } from "./codebase.ts";
|
||||
import { parseMetadataFileIfExists } from "./metadata.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
import { findCodebase } from "../commands/sync/sync.ts";
|
||||
import type { LocalScriptInfo } from "../../windmill-utils-internal/src/inline-scripts/replacer.ts";
|
||||
import type { RawScript } from "../../../gen/types.gen.ts";
|
||||
|
||||
export class UnsupportedLocalPathScriptPreviewError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "UnsupportedLocalPathScriptPreviewError";
|
||||
}
|
||||
}
|
||||
|
||||
async function readOptionalLock(scriptPath: string): Promise<string | undefined> {
|
||||
try {
|
||||
return await readFile(scriptPath + ".script.lock", "utf-8");
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeOptionalLock(lock: string | undefined): string | undefined {
|
||||
return typeof lock === "string" && lock.trim() === "" ? undefined : lock;
|
||||
}
|
||||
|
||||
async function bundleSingleFileCodebaseScript(
|
||||
filePath: string,
|
||||
codebase: SyncCodebase
|
||||
): Promise<string> {
|
||||
if (codebase.customBundler) {
|
||||
// Pass the script path as a positional shell argument so existing shell-based
|
||||
// custom bundlers still work without interpolating the path into the command.
|
||||
return execFileSync(
|
||||
"sh",
|
||||
["-lc", `${codebase.customBundler} "$1"`, "sh", filePath],
|
||||
{
|
||||
maxBuffer: 1024 * 1024 * 50,
|
||||
}
|
||||
).toString();
|
||||
}
|
||||
|
||||
const esbuild = await import("esbuild");
|
||||
const out = await esbuild.build({
|
||||
entryPoints: [filePath],
|
||||
// Inline rawscripts are executed through the standard module wrapper,
|
||||
// so the bundle must expose `main` as an ESM export.
|
||||
format: "esm",
|
||||
bundle: true,
|
||||
write: false,
|
||||
external: codebase.external,
|
||||
inject: codebase.inject,
|
||||
define: codebase.define,
|
||||
loader: codebase.loader ?? { ".node": "file" },
|
||||
outdir: "/",
|
||||
platform: "node",
|
||||
packages: "bundle",
|
||||
target: "esnext",
|
||||
banner: codebase.banner,
|
||||
});
|
||||
|
||||
if (out.outputFiles.length === 0) {
|
||||
throw new Error(`No output files found for ${filePath}`);
|
||||
}
|
||||
if (out.outputFiles.length > 1) {
|
||||
throw new UnsupportedLocalPathScriptPreviewError(
|
||||
`Local PathScript ${filePath} requires a multi-file bundle, which flow preview/dev cannot inline yet`
|
||||
);
|
||||
}
|
||||
if (Array.isArray(codebase.assets) && codebase.assets.length > 0) {
|
||||
throw new UnsupportedLocalPathScriptPreviewError(
|
||||
`Local PathScript ${filePath} requires codebase assets, which flow preview/dev cannot inline yet`
|
||||
);
|
||||
}
|
||||
|
||||
return out.outputFiles[0].text;
|
||||
}
|
||||
|
||||
export function createPreviewLocalScriptReader(opts: {
|
||||
exts: string[];
|
||||
defaultTs?: "bun" | "deno";
|
||||
codebases: SyncCodebase[];
|
||||
}): (scriptPath: string) => Promise<LocalScriptInfo | undefined> {
|
||||
return async (scriptPath) => {
|
||||
const localScript = await resolvePreviewLocalScriptState(scriptPath, opts);
|
||||
if (!localScript) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const content = localScript.codebase
|
||||
? await bundleSingleFileCodebaseScript(localScript.filePath, localScript.codebase)
|
||||
: localScript.content;
|
||||
|
||||
return {
|
||||
content,
|
||||
language: localScript.language,
|
||||
lock: localScript.lock,
|
||||
tag: localScript.tag,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
export type PreviewLocalScriptState = {
|
||||
filePath: string;
|
||||
content: string;
|
||||
language: RawScript["language"];
|
||||
lock?: string;
|
||||
tag?: string;
|
||||
codebase?: SyncCodebase;
|
||||
codebaseDigest?: string;
|
||||
};
|
||||
|
||||
export async function resolvePreviewLocalScriptState(
|
||||
scriptPath: string,
|
||||
opts: {
|
||||
exts: string[];
|
||||
defaultTs?: "bun" | "deno";
|
||||
codebases: SyncCodebase[];
|
||||
}
|
||||
): Promise<PreviewLocalScriptState | undefined> {
|
||||
for (const ext of opts.exts) {
|
||||
const filePath = scriptPath + ext;
|
||||
let fileStat;
|
||||
try {
|
||||
fileStat = await stat(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!fileStat.isFile()) continue;
|
||||
|
||||
const language = inferContentTypeFromFilePath(filePath, opts.defaultTs);
|
||||
const metadata = await parseMetadataFileIfExists(scriptPath);
|
||||
const rawLock = metadata?.payload?.lock ?? (await readOptionalLock(scriptPath));
|
||||
const codebase =
|
||||
language === "bun" ? findCodebase(filePath, opts.codebases) : undefined;
|
||||
|
||||
return {
|
||||
filePath,
|
||||
content: await readFile(filePath, "utf-8"),
|
||||
language,
|
||||
lock: normalizeOptionalLock(rawLock),
|
||||
tag: metadata?.payload?.tag,
|
||||
codebase,
|
||||
codebaseDigest: codebase
|
||||
? await codebase.getDigest(
|
||||
Array.isArray(codebase.assets) && codebase.assets.length > 0
|
||||
)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
+422
-104
@@ -5,7 +5,8 @@ import * as log from "../core/log.ts";
|
||||
import { stringify as yamlStringify } from "yaml";
|
||||
import { yamlParseFile } from "./yaml.ts";
|
||||
import { readFile, writeFile, stat, rm, readdir } from "node:fs/promises";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { readFileSync, existsSync, readdirSync, statSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import * as path from "node:path";
|
||||
import { createRequire } from "node:module";
|
||||
import {
|
||||
ScriptMetadata,
|
||||
@@ -15,19 +16,23 @@ import { Workspace } from "../commands/workspace/workspace.ts";
|
||||
import {
|
||||
ScriptLanguage,
|
||||
workspaceDependenciesLanguages,
|
||||
languageNeedsLock,
|
||||
} from "./script_common.ts";
|
||||
import { inferContentTypeFromFilePath } from "./script_common.ts";
|
||||
import { getModuleFolderSuffix, isModuleEntryPoint, getScriptBasePathFromModulePath } from "./resource_folders.ts";
|
||||
import { findCodebase, yamlOptions } from "../commands/sync/sync.ts";
|
||||
import { generateHash, readInlinePathSync, getHeaders } from "./utils.ts";
|
||||
|
||||
import { SyncCodebase } from "./codebase.ts";
|
||||
import { argSigToJsonSchemaType } from "../../windmill-utils-internal/src/parse/parse-schema.ts";
|
||||
import { getIsWin } from "./utils.ts";
|
||||
import { extractRelativeImports } from "./relative_imports.ts";
|
||||
import { DoubleLinkedDependencyTree } from "./dependency_tree.ts";
|
||||
|
||||
const _require = createRequire(import.meta.url);
|
||||
const _parserCache = new Map<string, Promise<any>>();
|
||||
|
||||
function loadParser(pkgName: string): Promise<any> {
|
||||
export function loadParser(pkgName: string): Promise<any> {
|
||||
let p = _parserCache.get(pkgName);
|
||||
if (!p) {
|
||||
p = (async () => {
|
||||
@@ -50,27 +55,28 @@ export class LockfileGenerationError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
export async function generateAllMetadata() {}
|
||||
|
||||
export async function getRawWorkspaceDependencies(): Promise<Record<string, string>> {
|
||||
export async function getRawWorkspaceDependencies(legacyBehaviour: boolean): Promise<Record<string, string>> {
|
||||
const rawWorkspaceDeps: Record<string, string> = {};
|
||||
|
||||
|
||||
try {
|
||||
const entries = await readdir("dependencies", { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (entry.isDirectory()) continue;
|
||||
|
||||
|
||||
const filePath = `dependencies/${entry.name}`;
|
||||
const content = await readFile(filePath, "utf-8");
|
||||
|
||||
|
||||
// Find matching language
|
||||
for (const lang of workspaceDependenciesLanguages) {
|
||||
if (entry.name.endsWith(lang.filename)) {
|
||||
// Check if out of sync
|
||||
const contentHash = await generateHash(content + filePath);
|
||||
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
|
||||
|
||||
if (!isUpToDate) {
|
||||
if (legacyBehaviour) {
|
||||
const contentHash = await generateHash(content + filePath);
|
||||
const isUpToDate = await checkifMetadataUptodate(filePath, contentHash, undefined);
|
||||
if (!isUpToDate) {
|
||||
rawWorkspaceDeps[filePath] = content;
|
||||
}
|
||||
} else {
|
||||
rawWorkspaceDeps[filePath] = content;
|
||||
}
|
||||
break;
|
||||
@@ -184,15 +190,22 @@ export async function generateScriptMetadataInternal(
|
||||
noStaleMessage: boolean,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
codebases: SyncCodebase[],
|
||||
justUpdateMetadataLock?: boolean
|
||||
justUpdateMetadataLock?: boolean,
|
||||
legacyBehaviour?: boolean,
|
||||
tree?: DoubleLinkedDependencyTree
|
||||
): Promise<string | undefined> {
|
||||
const remotePath = scriptPath
|
||||
.substring(0, scriptPath.indexOf("."))
|
||||
.replaceAll(SEP, "/");
|
||||
// Detect folder layout: my_script__mod/script.ts
|
||||
const isFolderLayout = isModuleEntryPoint(scriptPath);
|
||||
|
||||
// remotePath is the Windmill API path (e.g., "u/admin/my_script")
|
||||
const remotePath = isFolderLayout
|
||||
? getScriptBasePathFromModulePath(scriptPath)!.replaceAll(SEP, "/")
|
||||
: scriptPath.substring(0, scriptPath.indexOf(".")).replaceAll(SEP, "/");
|
||||
|
||||
const language = inferContentTypeFromFilePath(scriptPath, opts.defaultTs);
|
||||
|
||||
|
||||
// For folder layout, parseMetadataFile is called with remotePath which
|
||||
// will find __mod/script.yaml via the folder layout fallback
|
||||
const metadataWithType = await parseMetadataFile(
|
||||
remotePath,
|
||||
undefined,
|
||||
@@ -208,19 +221,73 @@ export async function generateScriptMetadataInternal(
|
||||
language
|
||||
);
|
||||
|
||||
// Compute the module folder path early so we can include module hashes in stale check
|
||||
const moduleFolderPath = isFolderLayout
|
||||
? path.dirname(scriptPath)
|
||||
: scriptPath.substring(0, scriptPath.indexOf(".")) + getModuleFolderSuffix();
|
||||
|
||||
// Note: rawWorkspaceDependencies are now passed in as parameter instead of being searched hierarchically
|
||||
let hash = await generateScriptHash(filteredRawWorkspaceDependencies, scriptContent, metadataContent);
|
||||
const hasModules = existsSync(moduleFolderPath) && statSync(moduleFolderPath).isDirectory();
|
||||
|
||||
if (await checkifMetadataUptodate(remotePath, hash, undefined)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
|
||||
);
|
||||
// In non-legacy mode, workspace deps are tracked via the tree — exclude from hash
|
||||
const depsForHash = (!legacyBehaviour && tree) ? {} : filteredRawWorkspaceDependencies;
|
||||
let hash = await generateScriptHash(depsForHash, scriptContent, metadataContent);
|
||||
|
||||
// Compute per-module hashes for stale detection (like flow inline scripts)
|
||||
let moduleHashes: Record<string, string> = {};
|
||||
if (hasModules) {
|
||||
moduleHashes = await computeModuleHashes(
|
||||
moduleFolderPath, opts.defaultTs, (!legacyBehaviour && tree) ? {} : rawWorkspaceDependencies, isFolderLayout
|
||||
);
|
||||
}
|
||||
const hasModuleHashes = Object.keys(moduleHashes).length > 0;
|
||||
|
||||
// If modules exist, combine main script hash + module hashes into a meta-hash
|
||||
let checkHash = hash;
|
||||
let checkSubpath: string | undefined;
|
||||
if (hasModuleHashes) {
|
||||
const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b));
|
||||
checkHash = await generateHash(hash + JSON.stringify(sortedEntries));
|
||||
checkSubpath = SCRIPT_TOP_HASH;
|
||||
}
|
||||
|
||||
const conf = await readLockfile();
|
||||
|
||||
// Use checkHash (includes module hashes) so module changes are detected as stale
|
||||
const isDirectlyStale = !(await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath));
|
||||
|
||||
// New behaviour: tree-based dependency tracking
|
||||
if (!legacyBehaviour && tree) {
|
||||
if (dryRun) {
|
||||
// First pass: populate tree with script and its imports
|
||||
const imports = await extractRelativeImports(scriptContent, remotePath, language);
|
||||
await tree.addNode(remotePath, scriptContent, language, metadataContent, imports, "script", remotePath, scriptPath, isDirectlyStale);
|
||||
return;
|
||||
}
|
||||
// Second pass: proceed to generate (caller verified this script is stale via tree)
|
||||
} else {
|
||||
// Legacy behaviour: use existing staleness check
|
||||
if (await checkifMetadataUptodate(remotePath, checkHash, conf, checkSubpath)) {
|
||||
if (!noStaleMessage) {
|
||||
log.info(
|
||||
colors.green(`Script ${remotePath} metadata is up-to-date, skipping`)
|
||||
);
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
let detail = `${remotePath} (${language})`;
|
||||
if (hasModuleHashes) {
|
||||
const changed: string[] = [];
|
||||
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
|
||||
if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) {
|
||||
changed.push(modulePath);
|
||||
}
|
||||
}
|
||||
if (changed.length > 0) {
|
||||
detail += ` [changed modules: ${changed.join(", ")}]`;
|
||||
}
|
||||
}
|
||||
return detail;
|
||||
}
|
||||
return;
|
||||
} else if (dryRun) {
|
||||
return `${remotePath} (${language})`;
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock && !noStaleMessage) {
|
||||
@@ -245,36 +312,97 @@ export async function generateScriptMetadataInternal(
|
||||
const hasCodebase = findCodebase(scriptPath, codebases) != undefined;
|
||||
|
||||
if (!hasCodebase) {
|
||||
const tempScriptRefs = tree?.getTempScriptRefs(remotePath);
|
||||
const lockPathOverride = isFolderLayout
|
||||
? path.dirname(scriptPath) + "/script.lock"
|
||||
: undefined;
|
||||
await updateScriptLock(
|
||||
workspace,
|
||||
scriptContent,
|
||||
language,
|
||||
remotePath,
|
||||
metadataParsedContent,
|
||||
filteredRawWorkspaceDependencies
|
||||
filteredRawWorkspaceDependencies,
|
||||
tempScriptRefs,
|
||||
lockPathOverride,
|
||||
);
|
||||
} else {
|
||||
metadataParsedContent.lock = "";
|
||||
}
|
||||
|
||||
// Generate locks for modules in __mod/ folder
|
||||
if (hasModules) {
|
||||
// Identify which modules changed by comparing per-module hashes
|
||||
let changedModules: string[] | undefined;
|
||||
if (hasModuleHashes) {
|
||||
changedModules = [];
|
||||
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
|
||||
if (!(await checkifMetadataUptodate(remotePath, moduleHash, conf, modulePath))) {
|
||||
changedModules.push(modulePath);
|
||||
}
|
||||
}
|
||||
if (changedModules.length === 0) {
|
||||
changedModules = undefined; // no modules changed, skip lock regeneration
|
||||
}
|
||||
}
|
||||
await updateModuleLocks(
|
||||
workspace, moduleFolderPath, "", remotePath,
|
||||
rawWorkspaceDependencies, opts.defaultTs, changedModules,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
|
||||
if (isFolderLayout) {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + getModuleFolderSuffix() + "/script.lock";
|
||||
} else {
|
||||
metadataParsedContent.lock =
|
||||
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
|
||||
}
|
||||
}
|
||||
let metaPath = remotePath + ".script.yaml";
|
||||
let newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
|
||||
if (metadataWithType.isJson) {
|
||||
metaPath = remotePath + ".script.json";
|
||||
newMetadataContent = JSON.stringify(metadataParsedContent);
|
||||
|
||||
// Write metadata back to the correct path
|
||||
let metaPath: string;
|
||||
let newMetadataContent: string;
|
||||
if (isFolderLayout) {
|
||||
if (metadataWithType.isJson) {
|
||||
metaPath = path.dirname(scriptPath) + "/script.json";
|
||||
newMetadataContent = JSON.stringify(metadataParsedContent);
|
||||
} else {
|
||||
metaPath = path.dirname(scriptPath) + "/script.yaml";
|
||||
newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
|
||||
}
|
||||
} else {
|
||||
if (metadataWithType.isJson) {
|
||||
metaPath = remotePath + ".script.json";
|
||||
newMetadataContent = JSON.stringify(metadataParsedContent);
|
||||
} else {
|
||||
metaPath = remotePath + ".script.yaml";
|
||||
newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
|
||||
}
|
||||
}
|
||||
|
||||
const metadataContentUsedForHash = newMetadataContent;
|
||||
|
||||
hash = await generateScriptHash(
|
||||
filteredRawWorkspaceDependencies,
|
||||
depsForHash,
|
||||
scriptContent,
|
||||
metadataContentUsedForHash
|
||||
);
|
||||
await updateMetadataGlobalLock(remotePath, hash);
|
||||
|
||||
// Store hashes in wmill-lock.yaml
|
||||
if (hasModuleHashes) {
|
||||
// Use per-module hash tracking (like flow inline scripts)
|
||||
const sortedEntries = Object.entries(moduleHashes).sort(([a], [b]) => a.localeCompare(b));
|
||||
const metaHash = await generateHash(hash + JSON.stringify(sortedEntries));
|
||||
await clearGlobalLock(remotePath);
|
||||
await updateMetadataGlobalLock(remotePath, metaHash, SCRIPT_TOP_HASH);
|
||||
for (const [modulePath, moduleHash] of Object.entries(moduleHashes)) {
|
||||
await updateMetadataGlobalLock(remotePath, moduleHash, modulePath);
|
||||
}
|
||||
} else {
|
||||
await updateMetadataGlobalLock(remotePath, hash);
|
||||
}
|
||||
|
||||
if (!justUpdateMetadataLock) {
|
||||
await writeFile(metaPath, newMetadataContent, "utf-8");
|
||||
}
|
||||
@@ -300,11 +428,9 @@ export async function updateScriptSchema(
|
||||
} else {
|
||||
delete metadataContent.has_preprocessor;
|
||||
}
|
||||
if (result.no_main_func) {
|
||||
metadataContent.no_main_func = result.no_main_func;
|
||||
} else {
|
||||
delete metadataContent.no_main_func;
|
||||
}
|
||||
// auto_kind is intentionally not written to metadata — it is auto-detected
|
||||
// by the parser at deploy time from script content.
|
||||
delete metadataContent.auto_kind;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -329,6 +455,7 @@ const LANG_ANNOTATION_CONFIG: Partial<
|
||||
nativets: { comment: "//", keyword: "package_json" },
|
||||
go: { comment: "//", keyword: "go_mod" },
|
||||
php: { comment: "//", keyword: "composer_json" },
|
||||
powershell: { comment: "#", keyword: "modules_json" },
|
||||
};
|
||||
|
||||
export function extractWorkspaceDepsAnnotation(
|
||||
@@ -410,6 +537,7 @@ export async function computeLockCacheKey(
|
||||
scriptContent: string,
|
||||
language: ScriptLanguage,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string> {
|
||||
const annotation = extractWorkspaceDepsAnnotation(scriptContent, language);
|
||||
const annotationStr = annotation
|
||||
@@ -417,7 +545,10 @@ export async function computeLockCacheKey(
|
||||
: "none";
|
||||
const sortedDepsKeys = Object.keys(rawWorkspaceDependencies).sort();
|
||||
const depsStr = sortedDepsKeys.map((k) => `${k}=${rawWorkspaceDependencies[k]}`).join(";");
|
||||
return await generateHash(`${language}|${annotationStr}|${depsStr}`);
|
||||
const tempRefsStr = tempScriptRefs
|
||||
? Object.keys(tempScriptRefs).sort().map((k) => `${k}=${tempScriptRefs[k]}`).join(";")
|
||||
: "";
|
||||
return await generateHash(`${language}|${annotationStr}|${depsStr}|${tempRefsStr}`);
|
||||
}
|
||||
|
||||
const lockCache = new Map<string, string>();
|
||||
@@ -432,13 +563,15 @@ async function fetchScriptLock(
|
||||
language: ScriptLanguage,
|
||||
remotePath: string,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>
|
||||
): Promise<string> {
|
||||
const hasRawDeps = Object.keys(rawWorkspaceDependencies).length > 0;
|
||||
const cacheKey = hasRawDeps
|
||||
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies)
|
||||
const hasTempRefs = tempScriptRefs && Object.keys(tempScriptRefs).length > 0;
|
||||
const cacheKey = (hasRawDeps || hasTempRefs)
|
||||
? await computeLockCacheKey(scriptContent, language, rawWorkspaceDependencies, tempScriptRefs)
|
||||
: undefined;
|
||||
if (cacheKey && lockCache.has(cacheKey)) {
|
||||
log.info(`Using cached lockfile for ${remotePath}`);
|
||||
log.debug(`Using cached lockfile for ${remotePath}`);
|
||||
return lockCache.get(cacheKey)!;
|
||||
}
|
||||
|
||||
@@ -463,6 +596,8 @@ async function fetchScriptLock(
|
||||
raw_workspace_dependencies: Object.keys(rawWorkspaceDependencies).length > 0
|
||||
? rawWorkspaceDependencies : null,
|
||||
entrypoint: remotePath,
|
||||
temp_script_refs: tempScriptRefs && Object.keys(tempScriptRefs).length > 0
|
||||
? tempScriptRefs : null,
|
||||
}),
|
||||
}
|
||||
);
|
||||
@@ -502,11 +637,14 @@ async function updateScriptLock(
|
||||
language: ScriptLanguage,
|
||||
remotePath: string,
|
||||
metadataContent: Record<string, any>,
|
||||
rawWorkspaceDependencies: Record<string, string>
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
tempScriptRefs?: Record<string, string>,
|
||||
lockPathOverride?: string,
|
||||
): Promise<void> {
|
||||
if (
|
||||
!(
|
||||
workspaceDependenciesLanguages.some((l) => l.language == language) ||
|
||||
(workspaceDependenciesLanguages.some((l) => l.language == language) &&
|
||||
language !== "powershell") ||
|
||||
language == "deno" ||
|
||||
language == "rust" ||
|
||||
language == "ansible"
|
||||
@@ -518,7 +656,7 @@ async function updateScriptLock(
|
||||
|
||||
if (Object.keys(rawWorkspaceDependencies).length > 0) {
|
||||
const dependencyPaths = Object.keys(rawWorkspaceDependencies).join(', ');
|
||||
log.info(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
|
||||
log.debug(`Generating script lock for ${remotePath} with raw workspace dependencies: ${dependencyPaths}`);
|
||||
}
|
||||
|
||||
const lock = await fetchScriptLock(
|
||||
@@ -527,9 +665,10 @@ async function updateScriptLock(
|
||||
language,
|
||||
remotePath,
|
||||
rawWorkspaceDependencies,
|
||||
tempScriptRefs
|
||||
);
|
||||
|
||||
const lockPath = remotePath + ".script.lock";
|
||||
const lockPath = lockPathOverride ?? remotePath + ".script.lock";
|
||||
if (lock != "") {
|
||||
await writeFile(lockPath, lock, "utf-8");
|
||||
metadataContent.lock = "!inline " + lockPath.replaceAll(SEP, "/");
|
||||
@@ -545,6 +684,82 @@ async function updateScriptLock(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate locks for all module files in a __mod/ directory.
|
||||
* Recursively walks the directory and generates a lock for each module
|
||||
* whose language requires one.
|
||||
*/
|
||||
async function updateModuleLocks(
|
||||
workspace: Workspace,
|
||||
dirPath: string,
|
||||
relPrefix: string,
|
||||
scriptRemotePath: string,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
changedModules?: string[],
|
||||
): Promise<void> {
|
||||
const entries = readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await updateModuleLocks(workspace, fullPath, relPath, scriptRemotePath, rawWorkspaceDependencies, defaultTs, changedModules);
|
||||
} else if (entry.isFile()
|
||||
&& !entry.name.endsWith(".lock")
|
||||
// In folder layout, skip entry point files (script.{ext}, script.yaml, script.json, script.lock)
|
||||
&& !(relPrefix === "" && entry.name.startsWith("script."))
|
||||
) {
|
||||
let modLanguage: ScriptLanguage;
|
||||
try {
|
||||
modLanguage = inferContentTypeFromFilePath(entry.name, defaultTs);
|
||||
} catch {
|
||||
continue; // skip files with unrecognized extensions
|
||||
}
|
||||
|
||||
if (!languageNeedsLock(modLanguage)) continue;
|
||||
|
||||
// Skip unchanged modules when per-module hash tracking is active
|
||||
if (changedModules) {
|
||||
const normalizedRelPath = normalizeLockPath(relPath);
|
||||
if (!changedModules.includes(normalizedRelPath)) continue;
|
||||
}
|
||||
|
||||
const moduleContent = readFileSync(fullPath, "utf-8");
|
||||
const moduleRemotePath = scriptRemotePath + "/" + relPath;
|
||||
|
||||
log.debug(`Generating lock for module ${relPath}`);
|
||||
|
||||
try {
|
||||
const lock = await fetchScriptLock(
|
||||
workspace,
|
||||
moduleContent,
|
||||
modLanguage,
|
||||
moduleRemotePath,
|
||||
rawWorkspaceDependencies,
|
||||
);
|
||||
|
||||
const baseName = entry.name.replace(/\.[^.]+$/, '');
|
||||
const lockPath = path.join(dirPath, baseName + ".lock");
|
||||
if (lock != "") {
|
||||
writeFileSync(lockPath, lock, "utf-8");
|
||||
} else {
|
||||
try {
|
||||
if (existsSync(lockPath)) {
|
||||
const { rm: rmAsync } = await import("node:fs/promises");
|
||||
await rmAsync(lockPath);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
log.info(colors.yellow(`Failed to generate lock for module ${relPath}: ${e}`));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// below functions copied from Windmill's FE inferArgs function. TODO: refactor //
|
||||
////////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -556,7 +771,7 @@ export async function inferSchema(
|
||||
): Promise<{
|
||||
schema: any;
|
||||
has_preprocessor: boolean | undefined;
|
||||
no_main_func: boolean | undefined;
|
||||
auto_kind: string | undefined;
|
||||
}> {
|
||||
let inferedSchema: any;
|
||||
if (language === "python3") {
|
||||
@@ -666,7 +881,7 @@ export async function inferSchema(
|
||||
return {
|
||||
schema: defaultScriptMetadata().schema,
|
||||
has_preprocessor: false,
|
||||
no_main_func: false,
|
||||
auto_kind: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -691,6 +906,11 @@ export async function inferSchema(
|
||||
|
||||
argSigToJsonSchemaType(arg.typ, currentSchema.properties[arg.name]);
|
||||
|
||||
// For T | T[] detection for debouncing arg accumulation
|
||||
if ((arg as any).otyp && (arg as any).otyp.includes('[') && (arg as any).otyp.includes('|')) {
|
||||
currentSchema.properties[arg.name].originalType = (arg as any).otyp
|
||||
}
|
||||
|
||||
currentSchema.properties[arg.name].default = arg.default;
|
||||
|
||||
if (!arg.has_default && !currentSchema.required.includes(arg.name)) {
|
||||
@@ -701,7 +921,7 @@ export async function inferSchema(
|
||||
return {
|
||||
schema: currentSchema,
|
||||
has_preprocessor: inferedSchema.has_preprocessor,
|
||||
no_main_func: inferedSchema.no_main_func,
|
||||
auto_kind: inferedSchema.auto_kind,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -737,6 +957,38 @@ export function replaceLock(o?: { lock?: string | string[] }) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function parseMetadataFileIfExists(
|
||||
scriptPath: string
|
||||
): Promise<{ isJson: boolean; payload: any; path: string } | undefined> {
|
||||
let metadataFilePath = scriptPath + ".script.json";
|
||||
try {
|
||||
await stat(metadataFilePath);
|
||||
const payload = JSON.parse(await readFile(metadataFilePath, "utf-8"));
|
||||
replaceLock(payload);
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload,
|
||||
isJson: true,
|
||||
};
|
||||
} catch {
|
||||
try {
|
||||
metadataFilePath = scriptPath + ".script.yaml";
|
||||
await stat(metadataFilePath);
|
||||
const payload: any = await yamlParseFile(metadataFilePath);
|
||||
replaceLock(payload);
|
||||
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload,
|
||||
isJson: false,
|
||||
};
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function parseMetadataFile(
|
||||
scriptPath: string,
|
||||
generateMetadataIfMissing:
|
||||
@@ -770,62 +1022,87 @@ export async function parseMetadataFile(
|
||||
isJson: false,
|
||||
};
|
||||
} catch {
|
||||
// no metadata file at all. Create it
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Creating script metadata file for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
metadataFilePath = scriptPath + ".script.yaml";
|
||||
let scriptInitialMetadata = defaultScriptMetadata();
|
||||
const lockPath = scriptPath + ".script.lock";
|
||||
scriptInitialMetadata.lock = "!inline " + lockPath;
|
||||
const scriptInitialMetadataYaml = yamlStringify(
|
||||
scriptInitialMetadata as Record<string, any>,
|
||||
yamlOptions
|
||||
);
|
||||
|
||||
await writeFile(metadataFilePath, scriptInitialMetadataYaml, { flag: "wx", encoding: "utf-8" });
|
||||
await writeFile(lockPath, "", { flag: "wx", encoding: "utf-8" });
|
||||
|
||||
if (generateMetadataIfMissing) {
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Generating lockfile and schema for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
// Try folder layout: {scriptPath}__mod/script.yaml or .json
|
||||
const moduleFolderMeta = scriptPath + getModuleFolderSuffix();
|
||||
try {
|
||||
metadataFilePath = moduleFolderMeta + "/script.json";
|
||||
await stat(metadataFilePath);
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload: JSON.parse(await readFile(metadataFilePath, "utf-8")),
|
||||
isJson: true,
|
||||
};
|
||||
} catch {
|
||||
try {
|
||||
await generateScriptMetadataInternal(
|
||||
generateMetadataIfMissing.path,
|
||||
generateMetadataIfMissing.workspaceRemote,
|
||||
generateMetadataIfMissing,
|
||||
false,
|
||||
false,
|
||||
generateMetadataIfMissing.rawWorkspaceDependencies,
|
||||
generateMetadataIfMissing.codebases,
|
||||
false
|
||||
);
|
||||
scriptInitialMetadata = (await yamlParseFile(
|
||||
metadataFilePath
|
||||
)) as ScriptMetadata;
|
||||
if (!generateMetadataIfMissing.schemaOnly) {
|
||||
replaceLock(scriptInitialMetadata);
|
||||
}
|
||||
} catch (e) {
|
||||
log.info(
|
||||
colors.yellow(
|
||||
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
|
||||
)
|
||||
);
|
||||
metadataFilePath = moduleFolderMeta + "/script.yaml";
|
||||
await stat(metadataFilePath);
|
||||
const payload: any = await yamlParseFile(metadataFilePath);
|
||||
replaceLock(payload);
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload,
|
||||
isJson: false,
|
||||
};
|
||||
} catch {
|
||||
// fall through to create metadata
|
||||
}
|
||||
}
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload: scriptInitialMetadata,
|
||||
isJson: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
// no metadata file at all. Create it
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Creating script metadata file for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
metadataFilePath = scriptPath + ".script.yaml";
|
||||
let scriptInitialMetadata = defaultScriptMetadata();
|
||||
const lockPath = scriptPath + ".script.lock";
|
||||
scriptInitialMetadata.lock = "!inline " + lockPath;
|
||||
const scriptInitialMetadataYaml = yamlStringify(
|
||||
scriptInitialMetadata as Record<string, any>,
|
||||
yamlOptions
|
||||
);
|
||||
|
||||
await writeFile(metadataFilePath, scriptInitialMetadataYaml, { flag: "wx", encoding: "utf-8" });
|
||||
await writeFile(lockPath, "", { flag: "wx", encoding: "utf-8" });
|
||||
|
||||
if (generateMetadataIfMissing) {
|
||||
log.info(
|
||||
(await blueColor())(
|
||||
`Generating lockfile and schema for ${metadataFilePath}`
|
||||
)
|
||||
);
|
||||
try {
|
||||
await generateScriptMetadataInternal(
|
||||
generateMetadataIfMissing.path,
|
||||
generateMetadataIfMissing.workspaceRemote,
|
||||
generateMetadataIfMissing,
|
||||
false,
|
||||
false,
|
||||
generateMetadataIfMissing.rawWorkspaceDependencies,
|
||||
generateMetadataIfMissing.codebases,
|
||||
false
|
||||
);
|
||||
scriptInitialMetadata = (await yamlParseFile(
|
||||
metadataFilePath
|
||||
)) as ScriptMetadata;
|
||||
if (!generateMetadataIfMissing.schemaOnly) {
|
||||
replaceLock(scriptInitialMetadata);
|
||||
}
|
||||
} catch (e) {
|
||||
log.info(
|
||||
colors.yellow(
|
||||
`Failed to generate lockfile and schema for ${metadataFilePath}: ${e}`
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
return {
|
||||
path: metadataFilePath,
|
||||
payload: scriptInitialMetadata,
|
||||
isJson: false,
|
||||
};
|
||||
}
|
||||
|
||||
interface Lock {
|
||||
@@ -834,6 +1111,7 @@ interface Lock {
|
||||
}
|
||||
|
||||
const WMILL_LOCKFILE = "wmill-lock.yaml";
|
||||
const SCRIPT_TOP_HASH = "__script_hash";
|
||||
|
||||
/**
|
||||
* Normalizes a path to use Linux separators (forward slashes).
|
||||
@@ -902,6 +1180,46 @@ export async function generateScriptHash(
|
||||
);
|
||||
}
|
||||
|
||||
async function computeModuleHashes(
|
||||
moduleFolderPath: string,
|
||||
defaultTs: "bun" | "deno" | undefined,
|
||||
rawWorkspaceDependencies: Record<string, string>,
|
||||
isFolderLayout: boolean,
|
||||
): Promise<Record<string, string>> {
|
||||
const hashes: Record<string, string> = {};
|
||||
|
||||
async function readDir(dirPath: string, relPrefix: string) {
|
||||
const entries = readdirSync(dirPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const fullPath = path.join(dirPath, entry.name);
|
||||
const relPath = relPrefix ? relPrefix + "/" + entry.name : entry.name;
|
||||
const isTopLevel = relPrefix === "";
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
await readDir(fullPath, relPath);
|
||||
} else if (
|
||||
entry.isFile() &&
|
||||
!entry.name.endsWith(".lock") &&
|
||||
!(isFolderLayout && isTopLevel && entry.name.startsWith("script."))
|
||||
) {
|
||||
try {
|
||||
inferContentTypeFromFilePath(entry.name, defaultTs);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const content = readFileSync(fullPath, "utf-8");
|
||||
const normalizedPath = normalizeLockPath(relPath);
|
||||
hashes[normalizedPath] = await generateHash(
|
||||
content + JSON.stringify(rawWorkspaceDependencies)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await readDir(moduleFolderPath, "");
|
||||
return hashes;
|
||||
}
|
||||
|
||||
export async function clearGlobalLock(path: string): Promise<void> {
|
||||
const conf = await readLockfile();
|
||||
if (!conf?.locks) {
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
/**
|
||||
* Relative Imports Utilities for CLI
|
||||
*
|
||||
* Provides functions to parse relative imports from TypeScript/Python scripts using WASM.
|
||||
*/
|
||||
|
||||
import { ScriptLanguage } from "./script_common.ts";
|
||||
import { loadParser } from "./metadata.ts";
|
||||
import * as log from "../core/log.ts";
|
||||
|
||||
/**
|
||||
* Extract relative imports from script content based on language.
|
||||
* Returns resolved absolute Windmill paths (e.g., "f/folder/helper").
|
||||
*/
|
||||
export async function extractRelativeImports(
|
||||
code: string,
|
||||
scriptPath: string,
|
||||
language: ScriptLanguage
|
||||
): Promise<string[]> {
|
||||
try {
|
||||
switch (language) {
|
||||
case "bun":
|
||||
case "nativets":
|
||||
case "deno": {
|
||||
const { parse_ts_relative_imports } = await loadParser("windmill-parser-wasm-ts");
|
||||
return parse_ts_relative_imports(code, scriptPath);
|
||||
}
|
||||
case "python3": {
|
||||
const { parse_py_relative_imports } = await loadParser("windmill-parser-wasm-py-imports");
|
||||
return parse_py_relative_imports(code, scriptPath);
|
||||
}
|
||||
default:
|
||||
return [];
|
||||
}
|
||||
} catch (e) {
|
||||
log.warn(`Failed to parse relative imports for ${scriptPath}: ${e}. Dependency tracking for relative imports will be disabled.`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -189,6 +189,26 @@ export function isFolderResourcePath(p: string): boolean {
|
||||
return isFlowPath(p) || isAppPath(p) || isRawAppPath(p);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path is inside a folder-based resource, checking BOTH dotted (.flow, .app, .raw_app)
|
||||
* and non-dotted (__flow, __app, __raw_app) formats regardless of the global nonDottedPaths setting.
|
||||
* Use this instead of isFolderResourcePath when the config may not yet be loaded or when
|
||||
* you need to handle mixed-format workspaces (e.g. generate-metadata scanning all files).
|
||||
*/
|
||||
export function isFolderResourcePathAnyFormat(p: string): boolean {
|
||||
const n = normalizeSep(p);
|
||||
for (const suffixes of [DOTTED_SUFFIXES, NON_DOTTED_SUFFIXES]) {
|
||||
if (
|
||||
n.includes(suffixes.flow + "/") ||
|
||||
n.includes(suffixes.app + "/") ||
|
||||
n.includes(suffixes.raw_app + "/")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect the resource type from a path, if any
|
||||
*/
|
||||
@@ -433,6 +453,66 @@ export function isRawAppFolderMetadataFile(p: string): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Script Module Path Functions
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* The suffix used for script module folders.
|
||||
* Unlike flows/apps, modules always use `__mod` (never dotted `.mod`)
|
||||
* to avoid confusion with file extensions.
|
||||
*/
|
||||
const MODULE_SUFFIX = "__mod";
|
||||
|
||||
/**
|
||||
* Get the module folder suffix (always "__mod")
|
||||
*/
|
||||
export function getModuleFolderSuffix(): string {
|
||||
return MODULE_SUFFIX;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a path is inside a script module folder.
|
||||
* Matches patterns like: .../my_script__mod/...
|
||||
*/
|
||||
export function isScriptModulePath(p: string): boolean {
|
||||
return normalizeSep(p).includes(MODULE_SUFFIX + "/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the module folder path from a script's base path (without extension).
|
||||
* e.g., "f/my_script" -> "f/my_script__mod"
|
||||
*/
|
||||
export function buildModuleFolderPath(scriptBasePath: string): string {
|
||||
return scriptBasePath + MODULE_SUFFIX;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file inside a __mod/ folder is the main entry point (script.{ext}).
|
||||
* Entry points are files named "script.*" directly under __mod/ (not in subdirs).
|
||||
*/
|
||||
export function isModuleEntryPoint(p: string): boolean {
|
||||
const norm = normalizeSep(p);
|
||||
const suffix = MODULE_SUFFIX + "/";
|
||||
const idx = norm.indexOf(suffix);
|
||||
if (idx === -1) return false;
|
||||
const rest = norm.slice(idx + suffix.length);
|
||||
return rest.startsWith("script.") && !rest.includes("/");
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the script base path from a module folder entry.
|
||||
* e.g., "u/admin/my_script__mod/script.ts" -> "u/admin/my_script"
|
||||
* e.g., "u/admin/my_script__mod/helper.ts" -> "u/admin/my_script"
|
||||
*/
|
||||
export function getScriptBasePathFromModulePath(p: string): string | undefined {
|
||||
const norm = normalizeSep(p);
|
||||
const suffix = MODULE_SUFFIX + "/";
|
||||
const idx = norm.indexOf(suffix);
|
||||
if (idx === -1) return undefined;
|
||||
return norm.slice(0, idx);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Sync-related Path Functions
|
||||
// ============================================================================
|
||||
|
||||
@@ -30,13 +30,15 @@ export type WorkspaceDependenciesLanguage =
|
||||
| { language: "bun", filename /** (raw requirements filename) */: "package.json" }
|
||||
| { language: "python3", filename: "requirements.in" }
|
||||
| { language: "php", filename: "composer.json" }
|
||||
| { language: "go", filename: "go.mod" };
|
||||
| { language: "go", filename: "go.mod" }
|
||||
| { language: "powershell", filename: "modules.json" };
|
||||
|
||||
export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
|
||||
{ language: "bun", filename: "package.json" },
|
||||
{ language: "python3", filename: "requirements.in" },
|
||||
{ language: "php", filename: "composer.json" },
|
||||
{ language: "go", filename: "go.mod" },
|
||||
{ language: "powershell", filename: "modules.json" },
|
||||
] as const;
|
||||
|
||||
/**
|
||||
@@ -45,7 +47,8 @@ export const workspaceDependenciesLanguages: WorkspaceDependenciesLanguage[] = [
|
||||
*/
|
||||
export function languageNeedsLock(language: ScriptLanguage | string): boolean {
|
||||
return (
|
||||
workspaceDependenciesLanguages.some((l) => l.language === language) ||
|
||||
(workspaceDependenciesLanguages.some((l) => l.language === language) &&
|
||||
language !== "powershell") ||
|
||||
language === "deno" ||
|
||||
language === "rust" ||
|
||||
language === "ansible"
|
||||
|
||||
Reference in New Issue
Block a user