starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-12 08:05:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(cli): improve codebase support + remove --stateful + warn when pushing stale metadata (#5139)

Browse Source
This commit is contained in:
Ruben Fiszel
2025-01-26 11:04:35 +01:00
committed by GitHub
parent 789c2ec4da
commit b47e9c14f2
5 changed files with 331 additions and 210 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli/metadata.ts
+44 -44
View File
@@ -49,7 +49,7 @@ import { FlowFile, replaceInlineScripts } from "./flow.ts";
import { getIsWin } from "./main.ts";
import { FlowValue } from "./gen/types.gen.ts";
export async function generateAllMetadata() {}
export async function generateAllMetadata() { }
function findClosestRawReqs(
lang: "bun" | "python3" | "php" | undefined,
@@ -104,7 +104,8 @@ export async function generateFlowLockInternal(
folder: string,
dryRun: boolean,
workspace: Workspace,
justUpdateMetadataLock?: boolean
justUpdateMetadataLock?: boolean,
noStaleMessage?: boolean
): Promise<string | undefined> {
if (folder.endsWith(SEP)) {
folder = folder.substring(0, folder.length - 1);
@@ -112,7 +113,7 @@ export async function generateFlowLockInternal(
const remote_path = folder
.replaceAll(SEP, "/")
.substring(0, folder.length - ".flow".length);
if (!justUpdateMetadataLock) {
if (!justUpdateMetadataLock && !noStaleMessage) {
log.info(`Generating lock for flow ${folder} at ${remote_path}`);
}
@@ -120,9 +121,11 @@ export async function generateFlowLockInternal(
const conf = await readLockfile();
if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) {
log.info(
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
);
if (!noStaleMessage) {
log.info(
colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`)
);
}
return;
} else if (dryRun) {
return remote_path;
@@ -222,12 +225,7 @@ export async function generateScriptMetadataInternal(
// read script content
const scriptContent = await Deno.readTextFile(scriptPath);
let metadataContent = await Deno.readTextFile(metadataWithType.path);
const c = findCodebase(scriptPath, codebases);
if (c) {
metadataContent += c.digest ?? "";
}
const metadataContent = await Deno.readTextFile(metadataWithType.path);
let hash = await generateScriptHash(rawReqs, scriptContent, metadataContent);
@@ -261,7 +259,9 @@ export async function generateScriptMetadataInternal(
}
if (!opts.schemaOnly && !justUpdateMetadataLock) {
if (!c) {
const hasCodebase = findCodebase(scriptPath, codebases) != undefined;
if (!hasCodebase) {
await updateScriptLock(
workspace,
scriptContent,
@@ -270,16 +270,13 @@ export async function generateScriptMetadataInternal(
metadataParsedContent,
rawReqs
);
metadataParsedContent.codebase = undefined;
} else {
metadataParsedContent.codebase = c.digest;
metadataParsedContent.lock = "";
metadataParsedContent.lock = '';
}
} else {
metadataParsedContent.lock =
"!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock";
}
let metaPath = remotePath + ".script.yaml";
let newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions);
if (metadataWithType.isJson) {
@@ -287,10 +284,8 @@ export async function generateScriptMetadataInternal(
newMetadataContent = JSON.stringify(metadataParsedContent);
}
let metadataContentUsedForHash = newMetadataContent;
if (c) {
metadataContentUsedForHash += c.digest ?? "";
}
const metadataContentUsedForHash = newMetadataContent;
hash = await generateScriptHash(
rawReqs,
scriptContent,
@@ -318,9 +313,9 @@ export async function updateScriptSchema(
path
);
metadataContent.schema = result.schema;
if (result.has_preprocessor != null)
if (result.has_preprocessor == true)
metadataContent.has_preprocessor = result.has_preprocessor;
if (result.no_main_func != null)
if (result.no_main_func === true)
metadataContent.no_main_func = result.no_main_func;
}
@@ -390,7 +385,7 @@ async function updateScriptLock(
if (await Deno.stat(lockPath)) {
await Deno.remove(lockPath);
}
} catch {}
} catch { }
metadataContent.lock = "";
}
} catch (e) {
@@ -431,7 +426,7 @@ export async function updateFlow(
} catch (e) {
try {
responseText = await rawResponse.text();
} catch {}
} catch { }
throw new Error(
`Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}`
);
@@ -528,8 +523,11 @@ export function inferSchema(
};
}
if (!currentSchema) {
currentSchema = {}
}
currentSchema.required = [];
const oldProperties = JSON.parse(JSON.stringify(currentSchema.properties));
const oldProperties = JSON.parse(JSON.stringify(currentSchema?.properties ?? {}));
currentSchema.properties = {};
for (const arg of inferedSchema.args) {
@@ -576,23 +574,23 @@ export function argSigToJsonSchemaType(
| string
| { resource: string | null }
| {
list:
| (string | { object: { key: string; typ: any }[] })
| { str: any }
| { object: { key: string; typ: any }[] }
| null;
}
list:
| (string | { object: { key: string; typ: any }[] })
| { str: any }
| { object: { key: string; typ: any }[] }
| null;
}
| { dynselect: string }
| { str: string[] | null }
| { object: { key: string; typ: any }[] }
| {
oneof: [
{
label: string;
properties: { key: string; typ: any }[];
}
];
},
oneof: [
{
label: string;
properties: { key: string; typ: any }[];
}
];
},
oldS: SchemaProperty
): void {
const newS: SchemaProperty = { type: "" };
@@ -795,10 +793,10 @@ export async function parseMetadataFile(
scriptPath: string,
generateMetadataIfMissing:
| (GlobalOptions & {
path: string;
workspaceRemote: Workspace;
schemaOnly?: boolean;
})
path: string;
workspaceRemote: Workspace;
schemaOnly?: boolean;
})
| undefined,
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
@@ -860,7 +858,9 @@ export async function parseMetadataFile(
scriptInitialMetadata = (await yamlParseFile(
metadataFilePath
)) as ScriptMetadata;
replaceLock(scriptInitialMetadata);
if (!generateMetadataIfMissing.schemaOnly) {
replaceLock(scriptInitialMetadata);
}
} catch (e) {
log.info(
colors.yellow(
cli/script.ts
+53 -42
View File
@@ -1,5 +1,5 @@
// deno-lint-ignore-file no-explicit-any
import { GlobalOptions, showDiff } from "./types.ts";
import { GlobalOptions } from "./types.ts";
import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
import {
colors,
@@ -24,7 +24,6 @@ import { Workspace } from "./workspace.ts";
import {
generateScriptMetadataInternal,
parseMetadataFile,
updateScriptSchema,
} from "./metadata.ts";
import {
ScriptLanguage,
@@ -34,6 +33,7 @@ import {
elementsToMap,
findCodebase,
readDirRecursiveWithIgnore,
Skips,
yamlOptions,
} from "./sync.ts";
import { ignoreF } from "./sync.ts";
@@ -119,7 +119,7 @@ export async function findResourceFile(path: string) {
if (validCandidates.length > 1) {
throw new Error(
"Found two resource files for the same resource" +
validCandidates.join(", ")
validCandidates.join(", ")
);
}
if (validCandidates.length < 1) {
@@ -162,7 +162,7 @@ export async function handleFile(
workspace: Workspace,
alreadySynced: string[],
message: string | undefined,
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" }) | undefined,
opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined,
globalDeps: GlobalDeps,
codebases: SyncCodebase[]
): Promise<boolean> {
@@ -238,21 +238,22 @@ export async function handleFile(
}
log.info(`Finished building the bundle for ${path}`);
}
const typed = (
await parseMetadataFile(
remotePath,
opts
? {
let typed =
opts?.skipScriptsMetadata ? undefined :
(await parseMetadataFile(
remotePath,
opts
? {
...opts,
path,
workspaceRemote: workspace,
schemaOnly: codebase ? true : undefined,
}
: undefined,
globalDeps,
codebases
)
)?.payload;
: undefined,
globalDeps,
codebases
)
)?.payload;
const workspaceId = workspace.workspaceId;
@@ -268,20 +269,29 @@ export async function handleFile(
}
const content = await Deno.readTextFile(path);
if (codebase) {
const typedBefore = JSON.parse(JSON.stringify(typed.schema));
await updateScriptSchema(content, language, typed, path);
if (typedBefore != typed.schema) {
log.info(`Updated metadata for bundle ${path}`);
showDiff(
yamlStringify(typedBefore, yamlOptions),
yamlStringify(typed.schema, yamlOptions)
);
await Deno.writeTextFile(
remotePath + ".script.yaml",
yamlStringify(typed as Record<string, any>, yamlOptions)
);
}
if (opts?.skipScriptsMetadata) {
// if (codebase) {
// const typedBefore = JSON.parse(JSON.stringify(typed.schema));
// await updateScriptSchema(content, language, typed, path);
// if (typedBefore != typed.schema) {
// log.info(`Updated metadata for bundle ${path}`);
// showDiff(
// yamlStringify(typedBefore, yamlOptions),
// yamlStringify(typed.schema, yamlOptions)
// );
// await Deno.writeTextFile(
// remotePath + ".script.yaml",
// yamlStringify(typed as Record<string, any>, yamlOptions)
// );
// }
// }
// else {
typed = structuredClone(remote);
// }
}
if (typed && codebase) {
typed.codebase = codebase.digest;
}
const requestBodyCommon: NewScript = {
@@ -312,6 +322,8 @@ export async function handleFile(
on_behalf_of_email: typed?.on_behalf_of_email,
};
// log.info(JSON.stringify(requestBodyCommon, null, 2))
// log.info(JSON.stringify(opts, null, 2))
if (remote) {
if (content === remote.content) {
if (
@@ -327,19 +339,19 @@ export async function handleFile(
deepEqual(typed.schema, remote.schema) &&
typed.tag == remote.tag &&
(typed.ws_error_handler_muted ?? false) ==
remote.ws_error_handler_muted &&
remote.ws_error_handler_muted &&
typed.dedicated_worker == remote.dedicated_worker &&
typed.cache_ttl == remote.cache_ttl &&
typed.concurrency_time_window_s ==
remote.concurrency_time_window_s &&
remote.concurrency_time_window_s &&
typed.concurrent_limit == remote.concurrent_limit &&
Boolean(typed.restart_unless_cancelled) ==
Boolean(remote.restart_unless_cancelled) &&
Boolean(remote.restart_unless_cancelled) &&
Boolean(typed.visible_to_runner_only) ==
Boolean(remote.visible_to_runner_only) &&
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
Boolean(typed.has_preprocessor) ==
Boolean(remote.has_preprocessor) &&
Boolean(remote.has_preprocessor) &&
typed.priority == Boolean(remote.priority) &&
typed.timeout == remote.timeout &&
//@ts-ignore
@@ -412,10 +424,9 @@ async function createScript(
workspace: workspaceId,
requestBody: body,
});
} catch (e) {
} catch (e: any) {
throw Error(
`Script creation for ${body.path} with parent ${
body.parent_hash
`Script creation for ${body.path} with parent ${body.parent_hash
} was not successful: ${e.body ?? e.message}`
);
}
@@ -441,8 +452,7 @@ async function createScript(
});
if (req.status != 201) {
throw Error(
`Script snapshot creation was not successful: ${req.status} - ${
req.statusText
`Script snapshot creation was not successful: ${req.status} - ${req.statusText
} - ${await req.text()}`
);
}
@@ -453,8 +463,8 @@ export async function findContentFile(filePath: string) {
const candidates = filePath.endsWith("script.json")
? exts.map((x) => filePath.replace(".script.json", x))
: filePath.endsWith("script.lock")
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
? exts.map((x) => filePath.replace(".script.lock", x))
: exts.map((x) => filePath.replace(".script.yaml", x));
const validCandidates = (
await Promise.all(
@@ -473,7 +483,7 @@ export async function findContentFile(filePath: string) {
if (validCandidates.length > 1) {
throw new Error(
"No content path given and more than one candidate found: " +
validCandidates.join(", ")
validCandidates.join(", ")
);
}
if (validCandidates.length < 1) {
@@ -858,6 +868,7 @@ async function generateMetadata(
} & SyncOptions,
scriptPath: string | undefined
) {
log.info("This command only works for workspace scripts, for flows inline scripts use `wmill flow generate-locks`");
if (scriptPath == "") {
scriptPath = undefined;
}
@@ -976,7 +987,7 @@ const command = new Command()
.action(bootstrap as any)
.command(
"generate-metadata",
"re-generate the metadata file updating the lock and the script schema"
"re-generate the metadata file updating the lock and the script schema (for flows, use `wmill flow generate-locks`)"
)
.arguments("[script:file]")
.option("--yes", "Skip confirmation prompt")
cli/sync.ts
+228 -120
View File
@@ -101,8 +101,9 @@ async function addCodebaseDigestIfRelevant(
return content;
}
let isTs = true;
const replacedPath = path.replace(".script.yaml", ".ts");
try {
await Deno.stat(path.replace(".script.yaml", ".ts"));
await Deno.stat(replacedPath);
} catch {
isTs = false;
}
@@ -110,12 +111,12 @@ async function addCodebaseDigestIfRelevant(
return content;
}
if (isTs) {
const c = findCodebase(path, codebases);
const c = findCodebase(replacedPath, codebases);
if (c) {
const parsed: any = yamlParseContent(path, content);
if (parsed && typeof parsed == "object") {
parsed["codebase"] = c.digest;
parsed["lock"] = undefined;
parsed["lock"] = '';
return yamlStringify(parsed, yamlOptions);
} else {
throw Error(
@@ -158,9 +159,8 @@ export async function FSFSElement(
// },
async getContentText(): Promise<string> {
const content = await Deno.readTextFile(localP);
const r = await addCodebaseDigestIfRelevant(localP, content, codebases);
// console.log(r);
const itemPath = localP.substring(p.length + 1)
const r = await addCodebaseDigestIfRelevant(itemPath, content, codebases);
return r;
},
};
@@ -352,12 +352,12 @@ function ZipFSElement(
)
? "flow"
: p.endsWith("app.json")
? "app"
: p.endsWith("script.json")
? "script"
: p.endsWith("resource.json")
? "resource"
: "other";
? "app"
: p.endsWith("script.json")
? "script"
: p.endsWith("resource.json")
? "resource"
: "other";
const isJson = p.endsWith(".json");
@@ -387,7 +387,7 @@ function ZipFSElement(
yield {
isDirectory: false,
path: path.join(finalPath, s.path),
async *getChildren() {},
async *getChildren() { },
// deno-lint-ignore require-await
async getContentText() {
return s.content;
@@ -398,7 +398,7 @@ function ZipFSElement(
yield {
isDirectory: false,
path: path.join(finalPath, "flow.yaml"),
async *getChildren() {},
async *getChildren() { },
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(flow, yamlOptions);
@@ -414,7 +414,7 @@ function ZipFSElement(
yield {
isDirectory: false,
path: path.join(finalPath, s.path),
async *getChildren() {},
async *getChildren() { },
// deno-lint-ignore require-await
async getContentText() {
return s.content;
@@ -425,7 +425,7 @@ function ZipFSElement(
yield {
isDirectory: false,
path: path.join(finalPath, "app.yaml"),
async *getChildren() {},
async *getChildren() { },
// deno-lint-ignore require-await
async getContentText() {
return yamlStringify(app, yamlOptions);
@@ -490,7 +490,7 @@ function ZipFSElement(
r.push({
isDirectory: false,
path: removeSuffix(finalPath, ".json") + ".lock",
async *getChildren() {},
async *getChildren() { },
// deno-lint-ignore require-await
async getContentText() {
return lock;
@@ -513,7 +513,7 @@ function ZipFSElement(
removeSuffix(finalPath, ".resource.json") +
".resource.file." +
formatExtension,
async *getChildren() {},
async *getChildren() { },
// deno-lint-ignore require-await
async getContentText() {
return fileContent;
@@ -574,19 +574,19 @@ export async function* readDirRecursiveWithIgnore(
// getContentBytes(): Promise<Uint8Array>;
getContentText(): Promise<string>;
}[] = [
{
path: root.path,
ignored: ignore(root.path, root.isDirectory),
isDirectory: root.isDirectory,
c: root.getChildren,
// getContentBytes(): Promise<Uint8Array> {
// throw undefined;
// },
getContentText(): Promise<string> {
throw undefined;
{
path: root.path,
ignored: ignore(root.path, root.isDirectory),
isDirectory: root.isDirectory,
c: root.getChildren,
// getContentBytes(): Promise<Uint8Array> {
// throw undefined;
// },
getContentText(): Promise<string> {
throw undefined;
},
},
},
];
];
while (stack.length > 0) {
const e = stack.pop()!;
@@ -606,7 +606,7 @@ export async function* readDirRecursiveWithIgnore(
type Added = { name: "added"; path: string; content: string };
type Deleted = { name: "deleted"; path: string };
type Edit = { name: "edited"; path: string; before: string; after: string };
type Edit = { name: "edited"; path: string; before: string; after: string; codebase?: string };
type Change = Added | Deleted | Edit;
@@ -649,7 +649,7 @@ export async function elementsToMap(
"js",
"lock",
"rs",
"cs",
"cs",
"yml",
].includes(path.split(".").pop() ?? "") &&
!isFileResource(path)
@@ -677,10 +677,11 @@ export async function elementsToMap(
return map;
}
interface Skips {
export interface Skips {
skipVariables?: boolean | undefined;
skipResources?: boolean | undefined;
skipSecrets?: boolean | undefined;
skipScriptsMetadata?: boolean | undefined;
includeSchedules?: boolean | undefined;
includeUsers?: boolean | undefined;
includeGroups?: boolean | undefined;
@@ -694,13 +695,14 @@ async function compareDynFSElement(
ignore: (path: string, isDirectory: boolean) => boolean,
json: boolean,
skips: Skips,
ignoreMetadataDeletion: boolean
ignoreMetadataDeletion: boolean,
codebases: SyncCodebase[]
): Promise<Change[]> {
const [m1, m2] = els2
? await Promise.all([
elementsToMap(els1, ignore, json, skips),
elementsToMap(els2, ignore, json, skips),
])
elementsToMap(els1, ignore, json, skips),
elementsToMap(els2, ignore, json, skips),
])
: [await elementsToMap(els1, ignore, json, skips), {}];
const changes: Change[] = [];
@@ -734,25 +736,81 @@ async function compareDynFSElement(
return yamlParseContent(k, v);
}
}
for (const [k, v] of Object.entries(m1)) {
const codebaseChanges: Record<string, string> = {};
for (let [k, v] of Object.entries(m1)) {
const isScriptMetadata = k.endsWith(".script.yaml") || k.endsWith(".script.json");
const skipMetadata = skips.skipScriptsMetadata && isScriptMetadata;
if (m2[k] === undefined) {
if (skipMetadata) {
continue;
}
changes.push({ name: "added", path: k, content: v });
} else if (
m2[k] != v &&
(!k.endsWith(".json") || !deepEqual(JSON.parse(v), JSON.parse(m2[k]))) &&
(!k.endsWith(".yaml") || !deepEqual(parseYaml(k, v), parseYaml(k, m2[k])))
) {
changes.push({ name: "edited", path: k, after: v, before: m2[k] });
} else {
if (m2[k] == v) {
continue;
}
else if (k.endsWith(".json")) {
if (deepEqual(JSON.parse(v), JSON.parse(m2[k]))) {
continue;
}
} else if (k.endsWith(".yaml")) {
const before = parseYaml(k, m2[k]);
const after = parseYaml(k, v);
if (deepEqual(before, after)) {
continue;
}
if (before.codebase != undefined) {
delete before.codebase;
m2[k] = yamlStringify(before, yamlOptions);
}
if (after.codebase != undefined) {
if (before.codebase != after.codebase) {
codebaseChanges[k] = after.codebase;
}
delete after.codebase;
v = yamlStringify(after, yamlOptions);
}
if (skipMetadata) {
continue;
}
}
changes.push({ name: "edited", path: k, after: v, before: m2[k], codebase: codebaseChanges[k] });
}
}
const remoteCodebase: Record<string, string> = {};
for (const [k] of Object.entries(m2)) {
if (
m1[k] === undefined &&
(!ignoreMetadataDeletion ||
(!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json")))
) {
changes.push({ name: "deleted", path: k });
if (m1[k] === undefined) {
if (!ignoreMetadataDeletion || (!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json"))) {
changes.push({ name: "deleted", path: k });
} else if (k?.endsWith(".script.yaml")) {
let o = parseYaml(k, m2[k]);
if (o.codebase != undefined) {
remoteCodebase[k] = o.codebase;
}
}
}
}
for (const [k, v] of Object.entries(remoteCodebase)) {
const tsFile = k.replace(".script.yaml", ".ts");
if (changes.find(c => c.path == tsFile && (c.name == "edited" || c.name == "deleted"))) {
continue;
}
let c = findCodebase(tsFile, codebases);
if (c?.digest != v) {
changes.push({ name: "edited", path: tsFile, codebase: v, before: m1[tsFile], after: m2[tsFile] });
}
}
for (const change of changes) {
const codebase = codebaseChanges[change.path];
if (!codebase) continue;
const tsFile = change.path.replace(".script.yaml", ".ts");
if (change.name == "edited" && change.path == tsFile) {
change.codebase = codebase;
}
}
@@ -887,6 +945,58 @@ export async function ignoreF(wmillconf: {
};
}
interface ChangeTracker {
scripts: string[];
flows: string[];
apps: string[];
}
// deno-lint-ignore no-inner-declarations
async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) {
const isScript = exts.some((e) => p.endsWith(e));
if (isScript) {
if (p.includes(".flow" + SEP)) {
const folder =
p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP;
if (!tracker.flows.includes(folder)) {
tracker.flows.push(folder);
}
} else if (p.includes(".app" + SEP)) {
const folder = p.substring(0, p.indexOf(".app" + SEP)) + ".app" + SEP;
if (!tracker.apps.includes(folder)) {
tracker.apps.push(folder);
}
} else {
if (!tracker.scripts.includes(p)) {
tracker.scripts.push(p);
}
}
} else if (p.endsWith(".script.yaml") || p.endsWith(".script.json")) {
try {
const contentPath = await findContentFile(p);
if (!contentPath) return;
if (tracker.scripts.includes(contentPath)) return;
tracker.scripts.push(contentPath);
} catch {
// ignore
}
}
}
async function buildTracker(changes: Change[]) {
const tracker: ChangeTracker = {
scripts: [],
flows: [],
apps: [],
};
for (const change of changes) {
if (change.name == "added" || change.name == "edited") {
await addToChangedIfNotExists(change.path, tracker);
}
}
return tracker;
}
export async function pull(opts: GlobalOptions & SyncOptions) {
opts = await mergeConfigWithConfigFile(opts);
@@ -940,7 +1050,8 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
await ignoreF(opts),
opts.json ?? false,
opts,
false
false,
codebases
);
log.info(
@@ -959,41 +1070,8 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
}
const conflicts = [];
const changedScripts: string[] = [];
const changedFlows: string[] = [];
const changedApps: string[] = [];
// deno-lint-ignore no-inner-declarations
async function addToChangedIfNotExists(p: string) {
const isScript = exts.some((e) => p.endsWith(e));
if (isScript) {
if (p.includes(".flow" + SEP)) {
const folder =
p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP;
if (!changedFlows.includes(folder)) {
changedFlows.push(folder);
}
} else if (p.includes(".app" + SEP)) {
const folder = p.substring(0, p.indexOf(".app" + SEP)) + ".app" + SEP;
if (!changedApps.includes(folder)) {
changedApps.push(folder);
}
} else {
if (!changedScripts.includes(p)) {
changedScripts.push(p);
}
}
} else if (p.endsWith(".script.yaml") || p.endsWith(".script.json")) {
try {
const contentPath = await findContentFile(p);
if (!contentPath) return;
if (changedScripts.includes(contentPath)) return;
changedScripts.push(contentPath);
} catch {
// ignore
}
}
}
log.info(colors.gray(`Applying changes to files ...`));
for await (const change of changes) {
@@ -1054,7 +1132,6 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
await ensureDir(path.dirname(stateTarget));
await Deno.copyFile(target, stateTarget);
}
await addToChangedIfNotExists(change.path);
} else if (change.name === "added") {
await ensureDir(path.dirname(target));
if (opts.stateful) {
@@ -1066,7 +1143,6 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
if (opts.stateful) {
await Deno.copyFile(target, stateTarget);
}
await addToChangedIfNotExists(change.path);
} else if (change.name === "deleted") {
try {
log.info(
@@ -1103,7 +1179,9 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
const globalDeps = await findGlobalDeps();
for (const change of changedScripts) {
const tracker: ChangeTracker = await buildTracker(changes);
for (const change of tracker.scripts) {
await generateScriptMetadataInternal(
change,
workspace,
@@ -1115,13 +1193,13 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
true
);
}
for (const change of changedFlows) {
for (const change of tracker.flows) {
log.info(`Updating lock for flow ${change}`);
await generateFlowLockInternal(change, false, workspace, true);
}
if (changedApps.length > 0) {
if (tracker.apps.length > 0) {
log.info(
`Apps ${changedApps.join(
`Apps ${tracker.apps.join(
", "
)} scripts were changed but ignoring for now`
);
@@ -1146,9 +1224,11 @@ function prettyChanges(changes: Change[]) {
);
} else if (change.name === "edited") {
log.info(
colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path)
colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path + (change.codebase ? ` (codebase changed)` : ""))
);
showDiff(change.before, change.after);
if (change.before != change.after) {
showDiff(change.before, change.after);
}
}
}
}
@@ -1238,9 +1318,58 @@ export async function push(opts: GlobalOptions & SyncOptions) {
await ignoreF(opts),
opts.json ?? false,
opts,
true
true,
codebases
);
const globalDeps = await findGlobalDeps();
const tracker: ChangeTracker = await buildTracker(changes);
const staleScripts: string[] = [];
const staleFlows: string[] = [];
for (const change of tracker.scripts) {
const stale = await generateScriptMetadataInternal(
change,
workspace,
opts,
true,
true,
globalDeps,
codebases,
false
);
if (stale) {
staleScripts.push(stale);
}
}
if (staleScripts.length > 0) {
log.info("")
log.warn("Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:");
for (const stale of staleScripts) {
log.warn(stale);
}
log.info("")
}
for (const change of tracker.flows) {
const stale = await generateFlowLockInternal(change, true, workspace, false, true);
if (stale) {
staleFlows.push(stale);
}
}
if (staleFlows.length > 0) {
log.warn("Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:");
for (const stale of staleFlows) {
log.warn(stale);
}
log.info("")
}
const version = await fetchVersion(workspace.remote);
log.info(colors.gray("Remote version: " + version));
@@ -1264,7 +1393,9 @@ export async function push(opts: GlobalOptions & SyncOptions) {
log.info(colors.gray(`Applying changes to files ...`));
const alreadySynced: string[] = [];
const globalDeps = await findGlobalDeps();
for await (const change of changes) {
const stateTarget = path.join(Deno.cwd(), ".wmill", change.path);
@@ -1512,24 +1643,13 @@ const command = new Command()
)
.command("pull")
.description("Pull any remote changes and apply them locally.")
.option(
"--fail-conflicts",
"Error on conflicts (both remote and local have changes on the same item)"
)
.option(
"--raw",
"Push without using state, just overwrite. (Default, has no effect)"
)
.option("--yes", "Pull without needing confirmation")
.option(
"--stateful",
"Pull using state tracking (create .wmill folder and needed for --fail-conflicts)"
)
.option("--plain-secrets", "Pull secrets as plain text")
.option("--json", "Use JSON instead of YAML")
.option("--skip-variables", "Skip syncing variables (including secrets)")
.option("--skip-secrets", "Skip syncing only secrets variables")
.option("--skip-resources", "Skip syncing resources")
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
.option("--include-schedules", "Include syncing schedules")
.option("--include-users", "Include syncing users")
.option("--include-groups", "Include syncing groups")
@@ -1551,25 +1671,13 @@ const command = new Command()
.action(pull as any)
.command("push")
.description("Push any local changes and apply them remotely.")
.option(
"--fail-conflicts",
"Error on conflicts (both remote and local have changes on the same item)"
)
.option(
"--raw",
"Push without using state, just overwrite. (Default, has no effect)"
)
.option(
"--stateful",
"Pull using state tracking (use .wmill folder and needed for --fail-conflicts)w"
)
.option("--skip-pull", "(stateful only) Push without pulling first")
.option("--yes", "Push without needing confirmation")
.option("--plain-secrets", "Push secrets as plain text")
.option("--json", "Use JSON instead of YAML")
.option("--skip-variables", "Skip syncing variables (including secrets)")
.option("--skip-secrets", "Skip syncing only secrets variables")
.option("--skip-resources", "Skip syncing resources")
// .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic")
.option("--include-schedules", "Include syncing schedules")
.option("--include-users", "Include syncing users")
.option("--include-groups", "Include syncing groups")
cli/types.ts
+4 -3
View File
@@ -79,7 +79,8 @@ export function showDiff(local: string, remote: string) {
log.info("Diff too large to display");
return;
}
for (const part of Diff.diffLines(local, remote)) {
for (const part of Diff.diffLines(local ?? '', remote ?? '')) {
if (part.removed) {
// print red if removed without newline
finalString += `\x1b[31m${part.value}\x1b[0m`;
@@ -157,8 +158,8 @@ export function parseFromPath(p: string, content: string): any {
return p.endsWith(".yaml")
? yamlParseContent(p, content)
: p.endsWith(".json")
? JSON.parse(content)
: content;
? JSON.parse(content)
: content;
}
export function parseFromFile(p: string): any {
if (p.endsWith(".json")) {
frontend/src/lib/hubPaths.json
+2 -1
View File
@@ -2,7 +2,8 @@
"gitSync_0": "hub/9087/sync-script-to-git-repo-windmill",
"gitSync_1": "hub/9987/sync-script-to-git-repo-windmill",
"gitSync_2": "hub/11498/sync-script-to-git-repo-windmill",
"gitSync": "hub/11533/sync-script-to-git-repo-windmill",
"gitSync_3": "hub/11533/sync-script-to-git-repo-windmill",
"gitSync": "hub/11580/sync-script-to-git-repo-windmill",
"gitSyncTest_0": "hub/9073/git-repo-test-read-write-windmill",
"gitSyncTest": "hub/11499/git-repo-test-read-write-windmill",
"slackErrorHandler": "hub/9206/workspace-or-schedule-error-handler-slack",