From b47e9c14f277339ae51fd3a40e00e8ba4e8a1390 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Sun, 26 Jan 2025 11:04:35 +0100 Subject: [PATCH] feat(cli): improve codebase support + remove --stateful + warn when pushing stale metadata (#5139) --- cli/metadata.ts | 88 ++++----- cli/script.ts | 95 +++++---- cli/sync.ts | 348 +++++++++++++++++++++------------ cli/types.ts | 7 +- frontend/src/lib/hubPaths.json | 3 +- 5 files changed, 331 insertions(+), 210 deletions(-) diff --git a/cli/metadata.ts b/cli/metadata.ts index 32cd556f76..e78b0bf539 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -49,7 +49,7 @@ import { FlowFile, replaceInlineScripts } from "./flow.ts"; import { getIsWin } from "./main.ts"; import { FlowValue } from "./gen/types.gen.ts"; -export async function generateAllMetadata() {} +export async function generateAllMetadata() { } function findClosestRawReqs( lang: "bun" | "python3" | "php" | undefined, @@ -104,7 +104,8 @@ export async function generateFlowLockInternal( folder: string, dryRun: boolean, workspace: Workspace, - justUpdateMetadataLock?: boolean + justUpdateMetadataLock?: boolean, + noStaleMessage?: boolean ): Promise { if (folder.endsWith(SEP)) { folder = folder.substring(0, folder.length - 1); @@ -112,7 +113,7 @@ export async function generateFlowLockInternal( const remote_path = folder .replaceAll(SEP, "/") .substring(0, folder.length - ".flow".length); - if (!justUpdateMetadataLock) { + if (!justUpdateMetadataLock && !noStaleMessage) { log.info(`Generating lock for flow ${folder} at ${remote_path}`); } @@ -120,9 +121,11 @@ export async function generateFlowLockInternal( const conf = await readLockfile(); if (await checkifMetadataUptodate(folder, hashes[TOP_HASH], conf, TOP_HASH)) { - log.info( - colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) - ); + if (!noStaleMessage) { + log.info( + colors.green(`Flow ${remote_path} metadata is up-to-date, skipping`) + ); + } return; } else if (dryRun) { return remote_path; @@ -222,12 +225,7 @@ export async function generateScriptMetadataInternal( // read script content const scriptContent = await Deno.readTextFile(scriptPath); - let metadataContent = await Deno.readTextFile(metadataWithType.path); - const c = findCodebase(scriptPath, codebases); - - if (c) { - metadataContent += c.digest ?? ""; - } + const metadataContent = await Deno.readTextFile(metadataWithType.path); let hash = await generateScriptHash(rawReqs, scriptContent, metadataContent); @@ -261,7 +259,9 @@ export async function generateScriptMetadataInternal( } if (!opts.schemaOnly && !justUpdateMetadataLock) { - if (!c) { + const hasCodebase = findCodebase(scriptPath, codebases) != undefined; + + if (!hasCodebase) { await updateScriptLock( workspace, scriptContent, @@ -270,16 +270,13 @@ export async function generateScriptMetadataInternal( metadataParsedContent, rawReqs ); - metadataParsedContent.codebase = undefined; } else { - metadataParsedContent.codebase = c.digest; - metadataParsedContent.lock = ""; + metadataParsedContent.lock = ''; } } else { metadataParsedContent.lock = "!inline " + remotePath.replaceAll(SEP, "/") + ".script.lock"; } - let metaPath = remotePath + ".script.yaml"; let newMetadataContent = yamlStringify(metadataParsedContent, yamlOptions); if (metadataWithType.isJson) { @@ -287,10 +284,8 @@ export async function generateScriptMetadataInternal( newMetadataContent = JSON.stringify(metadataParsedContent); } - let metadataContentUsedForHash = newMetadataContent; - if (c) { - metadataContentUsedForHash += c.digest ?? ""; - } + const metadataContentUsedForHash = newMetadataContent; + hash = await generateScriptHash( rawReqs, scriptContent, @@ -318,9 +313,9 @@ export async function updateScriptSchema( path ); metadataContent.schema = result.schema; - if (result.has_preprocessor != null) + if (result.has_preprocessor == true) metadataContent.has_preprocessor = result.has_preprocessor; - if (result.no_main_func != null) + if (result.no_main_func === true) metadataContent.no_main_func = result.no_main_func; } @@ -390,7 +385,7 @@ async function updateScriptLock( if (await Deno.stat(lockPath)) { await Deno.remove(lockPath); } - } catch {} + } catch { } metadataContent.lock = ""; } } catch (e) { @@ -431,7 +426,7 @@ export async function updateFlow( } catch (e) { try { responseText = await rawResponse.text(); - } catch {} + } catch { } throw new Error( `Failed to generate lockfile. Status was: ${rawResponse.statusText}, ${responseText}, ${e}` ); @@ -528,8 +523,11 @@ export function inferSchema( }; } + if (!currentSchema) { + currentSchema = {} + } currentSchema.required = []; - const oldProperties = JSON.parse(JSON.stringify(currentSchema.properties)); + const oldProperties = JSON.parse(JSON.stringify(currentSchema?.properties ?? {})); currentSchema.properties = {}; for (const arg of inferedSchema.args) { @@ -576,23 +574,23 @@ export function argSigToJsonSchemaType( | string | { resource: string | null } | { - list: - | (string | { object: { key: string; typ: any }[] }) - | { str: any } - | { object: { key: string; typ: any }[] } - | null; - } + list: + | (string | { object: { key: string; typ: any }[] }) + | { str: any } + | { object: { key: string; typ: any }[] } + | null; + } | { dynselect: string } | { str: string[] | null } | { object: { key: string; typ: any }[] } | { - oneof: [ - { - label: string; - properties: { key: string; typ: any }[]; - } - ]; - }, + oneof: [ + { + label: string; + properties: { key: string; typ: any }[]; + } + ]; + }, oldS: SchemaProperty ): void { const newS: SchemaProperty = { type: "" }; @@ -795,10 +793,10 @@ export async function parseMetadataFile( scriptPath: string, generateMetadataIfMissing: | (GlobalOptions & { - path: string; - workspaceRemote: Workspace; - schemaOnly?: boolean; - }) + path: string; + workspaceRemote: Workspace; + schemaOnly?: boolean; + }) | undefined, globalDeps: GlobalDeps, codebases: SyncCodebase[] @@ -860,7 +858,9 @@ export async function parseMetadataFile( scriptInitialMetadata = (await yamlParseFile( metadataFilePath )) as ScriptMetadata; - replaceLock(scriptInitialMetadata); + if (!generateMetadataIfMissing.schemaOnly) { + replaceLock(scriptInitialMetadata); + } } catch (e) { log.info( colors.yellow( diff --git a/cli/script.ts b/cli/script.ts index c341489534..e6b08e15da 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { GlobalOptions, showDiff } from "./types.ts"; +import { GlobalOptions } from "./types.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { colors, @@ -24,7 +24,6 @@ import { Workspace } from "./workspace.ts"; import { generateScriptMetadataInternal, parseMetadataFile, - updateScriptSchema, } from "./metadata.ts"; import { ScriptLanguage, @@ -34,6 +33,7 @@ import { elementsToMap, findCodebase, readDirRecursiveWithIgnore, + Skips, yamlOptions, } from "./sync.ts"; import { ignoreF } from "./sync.ts"; @@ -119,7 +119,7 @@ export async function findResourceFile(path: string) { if (validCandidates.length > 1) { throw new Error( "Found two resource files for the same resource" + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -162,7 +162,7 @@ export async function handleFile( workspace: Workspace, alreadySynced: string[], message: string | undefined, - opts: (GlobalOptions & { defaultTs?: "bun" | "deno" }) | undefined, + opts: (GlobalOptions & { defaultTs?: "bun" | "deno" } & Skips) | undefined, globalDeps: GlobalDeps, codebases: SyncCodebase[] ): Promise { @@ -238,21 +238,22 @@ export async function handleFile( } log.info(`Finished building the bundle for ${path}`); } - const typed = ( - await parseMetadataFile( - remotePath, - opts - ? { + let typed = + opts?.skipScriptsMetadata ? undefined : + (await parseMetadataFile( + remotePath, + opts + ? { ...opts, path, workspaceRemote: workspace, schemaOnly: codebase ? true : undefined, } - : undefined, - globalDeps, - codebases - ) - )?.payload; + : undefined, + globalDeps, + codebases + ) + )?.payload; const workspaceId = workspace.workspaceId; @@ -268,20 +269,29 @@ export async function handleFile( } const content = await Deno.readTextFile(path); - if (codebase) { - const typedBefore = JSON.parse(JSON.stringify(typed.schema)); - await updateScriptSchema(content, language, typed, path); - if (typedBefore != typed.schema) { - log.info(`Updated metadata for bundle ${path}`); - showDiff( - yamlStringify(typedBefore, yamlOptions), - yamlStringify(typed.schema, yamlOptions) - ); - await Deno.writeTextFile( - remotePath + ".script.yaml", - yamlStringify(typed as Record, yamlOptions) - ); - } + if (opts?.skipScriptsMetadata) { + // if (codebase) { + // const typedBefore = JSON.parse(JSON.stringify(typed.schema)); + // await updateScriptSchema(content, language, typed, path); + // if (typedBefore != typed.schema) { + // log.info(`Updated metadata for bundle ${path}`); + // showDiff( + // yamlStringify(typedBefore, yamlOptions), + // yamlStringify(typed.schema, yamlOptions) + // ); + // await Deno.writeTextFile( + // remotePath + ".script.yaml", + // yamlStringify(typed as Record, yamlOptions) + // ); + // } + // } + // else { + typed = structuredClone(remote); + // } + } + + if (typed && codebase) { + typed.codebase = codebase.digest; } const requestBodyCommon: NewScript = { @@ -312,6 +322,8 @@ export async function handleFile( on_behalf_of_email: typed?.on_behalf_of_email, }; + // log.info(JSON.stringify(requestBodyCommon, null, 2)) + // log.info(JSON.stringify(opts, null, 2)) if (remote) { if (content === remote.content) { if ( @@ -327,19 +339,19 @@ export async function handleFile( deepEqual(typed.schema, remote.schema) && typed.tag == remote.tag && (typed.ws_error_handler_muted ?? false) == - remote.ws_error_handler_muted && + remote.ws_error_handler_muted && typed.dedicated_worker == remote.dedicated_worker && typed.cache_ttl == remote.cache_ttl && typed.concurrency_time_window_s == - remote.concurrency_time_window_s && + remote.concurrency_time_window_s && typed.concurrent_limit == remote.concurrent_limit && Boolean(typed.restart_unless_cancelled) == - Boolean(remote.restart_unless_cancelled) && + Boolean(remote.restart_unless_cancelled) && Boolean(typed.visible_to_runner_only) == - Boolean(remote.visible_to_runner_only) && + Boolean(remote.visible_to_runner_only) && Boolean(typed.no_main_func) == Boolean(remote.no_main_func) && Boolean(typed.has_preprocessor) == - Boolean(remote.has_preprocessor) && + Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && typed.timeout == remote.timeout && //@ts-ignore @@ -412,10 +424,9 @@ async function createScript( workspace: workspaceId, requestBody: body, }); - } catch (e) { + } catch (e: any) { throw Error( - `Script creation for ${body.path} with parent ${ - body.parent_hash + `Script creation for ${body.path} with parent ${body.parent_hash } was not successful: ${e.body ?? e.message}` ); } @@ -441,8 +452,7 @@ async function createScript( }); if (req.status != 201) { throw Error( - `Script snapshot creation was not successful: ${req.status} - ${ - req.statusText + `Script snapshot creation was not successful: ${req.status} - ${req.statusText } - ${await req.text()}` ); } @@ -453,8 +463,8 @@ export async function findContentFile(filePath: string) { const candidates = filePath.endsWith("script.json") ? exts.map((x) => filePath.replace(".script.json", x)) : filePath.endsWith("script.lock") - ? exts.map((x) => filePath.replace(".script.lock", x)) - : exts.map((x) => filePath.replace(".script.yaml", x)); + ? exts.map((x) => filePath.replace(".script.lock", x)) + : exts.map((x) => filePath.replace(".script.yaml", x)); const validCandidates = ( await Promise.all( @@ -473,7 +483,7 @@ export async function findContentFile(filePath: string) { if (validCandidates.length > 1) { throw new Error( "No content path given and more than one candidate found: " + - validCandidates.join(", ") + validCandidates.join(", ") ); } if (validCandidates.length < 1) { @@ -858,6 +868,7 @@ async function generateMetadata( } & SyncOptions, scriptPath: string | undefined ) { + log.info("This command only works for workspace scripts, for flows inline scripts use `wmill flow generate-locks`"); if (scriptPath == "") { scriptPath = undefined; } @@ -976,7 +987,7 @@ const command = new Command() .action(bootstrap as any) .command( "generate-metadata", - "re-generate the metadata file updating the lock and the script schema" + "re-generate the metadata file updating the lock and the script schema (for flows, use `wmill flow generate-locks`)" ) .arguments("[script:file]") .option("--yes", "Skip confirmation prompt") diff --git a/cli/sync.ts b/cli/sync.ts index 408c01884e..e8d6bb4d1a 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -101,8 +101,9 @@ async function addCodebaseDigestIfRelevant( return content; } let isTs = true; + const replacedPath = path.replace(".script.yaml", ".ts"); try { - await Deno.stat(path.replace(".script.yaml", ".ts")); + await Deno.stat(replacedPath); } catch { isTs = false; } @@ -110,12 +111,12 @@ async function addCodebaseDigestIfRelevant( return content; } if (isTs) { - const c = findCodebase(path, codebases); + const c = findCodebase(replacedPath, codebases); if (c) { const parsed: any = yamlParseContent(path, content); if (parsed && typeof parsed == "object") { parsed["codebase"] = c.digest; - parsed["lock"] = undefined; + parsed["lock"] = ''; return yamlStringify(parsed, yamlOptions); } else { throw Error( @@ -158,9 +159,8 @@ export async function FSFSElement( // }, async getContentText(): Promise { const content = await Deno.readTextFile(localP); - - const r = await addCodebaseDigestIfRelevant(localP, content, codebases); - // console.log(r); + const itemPath = localP.substring(p.length + 1) + const r = await addCodebaseDigestIfRelevant(itemPath, content, codebases); return r; }, }; @@ -352,12 +352,12 @@ function ZipFSElement( ) ? "flow" : p.endsWith("app.json") - ? "app" - : p.endsWith("script.json") - ? "script" - : p.endsWith("resource.json") - ? "resource" - : "other"; + ? "app" + : p.endsWith("script.json") + ? "script" + : p.endsWith("resource.json") + ? "resource" + : "other"; const isJson = p.endsWith(".json"); @@ -387,7 +387,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, s.path), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return s.content; @@ -398,7 +398,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, "flow.yaml"), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return yamlStringify(flow, yamlOptions); @@ -414,7 +414,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, s.path), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return s.content; @@ -425,7 +425,7 @@ function ZipFSElement( yield { isDirectory: false, path: path.join(finalPath, "app.yaml"), - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return yamlStringify(app, yamlOptions); @@ -490,7 +490,7 @@ function ZipFSElement( r.push({ isDirectory: false, path: removeSuffix(finalPath, ".json") + ".lock", - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return lock; @@ -513,7 +513,7 @@ function ZipFSElement( removeSuffix(finalPath, ".resource.json") + ".resource.file." + formatExtension, - async *getChildren() {}, + async *getChildren() { }, // deno-lint-ignore require-await async getContentText() { return fileContent; @@ -574,19 +574,19 @@ export async function* readDirRecursiveWithIgnore( // getContentBytes(): Promise; getContentText(): Promise; }[] = [ - { - path: root.path, - ignored: ignore(root.path, root.isDirectory), - isDirectory: root.isDirectory, - c: root.getChildren, - // getContentBytes(): Promise { - // throw undefined; - // }, - getContentText(): Promise { - throw undefined; + { + path: root.path, + ignored: ignore(root.path, root.isDirectory), + isDirectory: root.isDirectory, + c: root.getChildren, + // getContentBytes(): Promise { + // throw undefined; + // }, + getContentText(): Promise { + throw undefined; + }, }, - }, - ]; + ]; while (stack.length > 0) { const e = stack.pop()!; @@ -606,7 +606,7 @@ export async function* readDirRecursiveWithIgnore( type Added = { name: "added"; path: string; content: string }; type Deleted = { name: "deleted"; path: string }; -type Edit = { name: "edited"; path: string; before: string; after: string }; +type Edit = { name: "edited"; path: string; before: string; after: string; codebase?: string }; type Change = Added | Deleted | Edit; @@ -649,7 +649,7 @@ export async function elementsToMap( "js", "lock", "rs", - "cs", + "cs", "yml", ].includes(path.split(".").pop() ?? "") && !isFileResource(path) @@ -677,10 +677,11 @@ export async function elementsToMap( return map; } -interface Skips { +export interface Skips { skipVariables?: boolean | undefined; skipResources?: boolean | undefined; skipSecrets?: boolean | undefined; + skipScriptsMetadata?: boolean | undefined; includeSchedules?: boolean | undefined; includeUsers?: boolean | undefined; includeGroups?: boolean | undefined; @@ -694,13 +695,14 @@ async function compareDynFSElement( ignore: (path: string, isDirectory: boolean) => boolean, json: boolean, skips: Skips, - ignoreMetadataDeletion: boolean + ignoreMetadataDeletion: boolean, + codebases: SyncCodebase[] ): Promise { const [m1, m2] = els2 ? await Promise.all([ - elementsToMap(els1, ignore, json, skips), - elementsToMap(els2, ignore, json, skips), - ]) + elementsToMap(els1, ignore, json, skips), + elementsToMap(els2, ignore, json, skips), + ]) : [await elementsToMap(els1, ignore, json, skips), {}]; const changes: Change[] = []; @@ -734,25 +736,81 @@ async function compareDynFSElement( return yamlParseContent(k, v); } } - for (const [k, v] of Object.entries(m1)) { + const codebaseChanges: Record = {}; + for (let [k, v] of Object.entries(m1)) { + const isScriptMetadata = k.endsWith(".script.yaml") || k.endsWith(".script.json"); + const skipMetadata = skips.skipScriptsMetadata && isScriptMetadata; + if (m2[k] === undefined) { + if (skipMetadata) { + continue; + } changes.push({ name: "added", path: k, content: v }); - } else if ( - m2[k] != v && - (!k.endsWith(".json") || !deepEqual(JSON.parse(v), JSON.parse(m2[k]))) && - (!k.endsWith(".yaml") || !deepEqual(parseYaml(k, v), parseYaml(k, m2[k]))) - ) { - changes.push({ name: "edited", path: k, after: v, before: m2[k] }); + } else { + if (m2[k] == v) { + continue; + } + else if (k.endsWith(".json")) { + if (deepEqual(JSON.parse(v), JSON.parse(m2[k]))) { + continue; + } + } else if (k.endsWith(".yaml")) { + const before = parseYaml(k, m2[k]); + const after = parseYaml(k, v); + if (deepEqual(before, after)) { + continue; + } + if (before.codebase != undefined) { + delete before.codebase; + m2[k] = yamlStringify(before, yamlOptions); + } + if (after.codebase != undefined) { + if (before.codebase != after.codebase) { + codebaseChanges[k] = after.codebase; + } + delete after.codebase; + v = yamlStringify(after, yamlOptions); + } + if (skipMetadata) { + continue; + } + } + changes.push({ name: "edited", path: k, after: v, before: m2[k], codebase: codebaseChanges[k] }); } } + const remoteCodebase: Record = {}; for (const [k] of Object.entries(m2)) { - if ( - m1[k] === undefined && - (!ignoreMetadataDeletion || - (!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json"))) - ) { - changes.push({ name: "deleted", path: k }); + if (m1[k] === undefined) { + if (!ignoreMetadataDeletion || (!k?.endsWith(".script.yaml") && !k?.endsWith(".script.json"))) { + changes.push({ name: "deleted", path: k }); + } else if (k?.endsWith(".script.yaml")) { + let o = parseYaml(k, m2[k]); + if (o.codebase != undefined) { + remoteCodebase[k] = o.codebase; + } + } + } + } + + for (const [k, v] of Object.entries(remoteCodebase)) { + const tsFile = k.replace(".script.yaml", ".ts"); + if (changes.find(c => c.path == tsFile && (c.name == "edited" || c.name == "deleted"))) { + continue; + } + let c = findCodebase(tsFile, codebases); + if (c?.digest != v) { + changes.push({ name: "edited", path: tsFile, codebase: v, before: m1[tsFile], after: m2[tsFile] }); + } + } + + for (const change of changes) { + const codebase = codebaseChanges[change.path]; + if (!codebase) continue; + + const tsFile = change.path.replace(".script.yaml", ".ts"); + if (change.name == "edited" && change.path == tsFile) { + change.codebase = codebase; } } @@ -887,6 +945,58 @@ export async function ignoreF(wmillconf: { }; } +interface ChangeTracker { + scripts: string[]; + flows: string[]; + apps: string[]; +} + +// deno-lint-ignore no-inner-declarations +async function addToChangedIfNotExists(p: string, tracker: ChangeTracker) { + const isScript = exts.some((e) => p.endsWith(e)); + if (isScript) { + if (p.includes(".flow" + SEP)) { + const folder = + p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP; + if (!tracker.flows.includes(folder)) { + tracker.flows.push(folder); + } + } else if (p.includes(".app" + SEP)) { + const folder = p.substring(0, p.indexOf(".app" + SEP)) + ".app" + SEP; + if (!tracker.apps.includes(folder)) { + tracker.apps.push(folder); + } + } else { + if (!tracker.scripts.includes(p)) { + tracker.scripts.push(p); + } + } + } else if (p.endsWith(".script.yaml") || p.endsWith(".script.json")) { + try { + const contentPath = await findContentFile(p); + if (!contentPath) return; + if (tracker.scripts.includes(contentPath)) return; + tracker.scripts.push(contentPath); + } catch { + // ignore + } + } +} + +async function buildTracker(changes: Change[]) { + const tracker: ChangeTracker = { + scripts: [], + flows: [], + apps: [], + }; + for (const change of changes) { + if (change.name == "added" || change.name == "edited") { + await addToChangedIfNotExists(change.path, tracker); + } + } + return tracker; +} + export async function pull(opts: GlobalOptions & SyncOptions) { opts = await mergeConfigWithConfigFile(opts); @@ -940,7 +1050,8 @@ export async function pull(opts: GlobalOptions & SyncOptions) { await ignoreF(opts), opts.json ?? false, opts, - false + false, + codebases ); log.info( @@ -959,41 +1070,8 @@ export async function pull(opts: GlobalOptions & SyncOptions) { } const conflicts = []; - const changedScripts: string[] = []; - const changedFlows: string[] = []; - const changedApps: string[] = []; - // deno-lint-ignore no-inner-declarations - async function addToChangedIfNotExists(p: string) { - const isScript = exts.some((e) => p.endsWith(e)); - if (isScript) { - if (p.includes(".flow" + SEP)) { - const folder = - p.substring(0, p.indexOf(".flow" + SEP)) + ".flow" + SEP; - if (!changedFlows.includes(folder)) { - changedFlows.push(folder); - } - } else if (p.includes(".app" + SEP)) { - const folder = p.substring(0, p.indexOf(".app" + SEP)) + ".app" + SEP; - if (!changedApps.includes(folder)) { - changedApps.push(folder); - } - } else { - if (!changedScripts.includes(p)) { - changedScripts.push(p); - } - } - } else if (p.endsWith(".script.yaml") || p.endsWith(".script.json")) { - try { - const contentPath = await findContentFile(p); - if (!contentPath) return; - if (changedScripts.includes(contentPath)) return; - changedScripts.push(contentPath); - } catch { - // ignore - } - } - } + log.info(colors.gray(`Applying changes to files ...`)); for await (const change of changes) { @@ -1054,7 +1132,6 @@ export async function pull(opts: GlobalOptions & SyncOptions) { await ensureDir(path.dirname(stateTarget)); await Deno.copyFile(target, stateTarget); } - await addToChangedIfNotExists(change.path); } else if (change.name === "added") { await ensureDir(path.dirname(target)); if (opts.stateful) { @@ -1066,7 +1143,6 @@ export async function pull(opts: GlobalOptions & SyncOptions) { if (opts.stateful) { await Deno.copyFile(target, stateTarget); } - await addToChangedIfNotExists(change.path); } else if (change.name === "deleted") { try { log.info( @@ -1103,7 +1179,9 @@ export async function pull(opts: GlobalOptions & SyncOptions) { const globalDeps = await findGlobalDeps(); - for (const change of changedScripts) { + const tracker: ChangeTracker = await buildTracker(changes); + + for (const change of tracker.scripts) { await generateScriptMetadataInternal( change, workspace, @@ -1115,13 +1193,13 @@ export async function pull(opts: GlobalOptions & SyncOptions) { true ); } - for (const change of changedFlows) { + for (const change of tracker.flows) { log.info(`Updating lock for flow ${change}`); await generateFlowLockInternal(change, false, workspace, true); } - if (changedApps.length > 0) { + if (tracker.apps.length > 0) { log.info( - `Apps ${changedApps.join( + `Apps ${tracker.apps.join( ", " )} scripts were changed but ignoring for now` ); @@ -1146,9 +1224,11 @@ function prettyChanges(changes: Change[]) { ); } else if (change.name === "edited") { log.info( - colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path) + colors.yellow(`~ ${getTypeStrFromPath(change.path)} ` + change.path + (change.codebase ? ` (codebase changed)` : "")) ); - showDiff(change.before, change.after); + if (change.before != change.after) { + showDiff(change.before, change.after); + } } } } @@ -1238,9 +1318,58 @@ export async function push(opts: GlobalOptions & SyncOptions) { await ignoreF(opts), opts.json ?? false, opts, - true + true, + codebases ); + + const globalDeps = await findGlobalDeps(); + + const tracker: ChangeTracker = await buildTracker(changes); + + const staleScripts: string[] = []; + const staleFlows: string[] = []; + for (const change of tracker.scripts) { + const stale = await generateScriptMetadataInternal( + change, + workspace, + opts, + true, + true, + globalDeps, + codebases, + false + ); + if (stale) { + staleScripts.push(stale); + } + } + + if (staleScripts.length > 0) { + log.info("") + log.warn("Stale scripts metadata found, you may want to update them using 'wmill script generate-metadata' before pushing:"); + for (const stale of staleScripts) { + log.warn(stale); + } + + log.info("") + } + + for (const change of tracker.flows) { + const stale = await generateFlowLockInternal(change, true, workspace, false, true); + if (stale) { + staleFlows.push(stale); + } + } + + if (staleFlows.length > 0) { + log.warn("Stale flows locks found, you may want to update them using 'wmill flow generate-locks' before pushing:"); + for (const stale of staleFlows) { + log.warn(stale); + } + log.info("") + } + const version = await fetchVersion(workspace.remote); log.info(colors.gray("Remote version: " + version)); @@ -1264,7 +1393,9 @@ export async function push(opts: GlobalOptions & SyncOptions) { log.info(colors.gray(`Applying changes to files ...`)); const alreadySynced: string[] = []; - const globalDeps = await findGlobalDeps(); + + + for await (const change of changes) { const stateTarget = path.join(Deno.cwd(), ".wmill", change.path); @@ -1512,24 +1643,13 @@ const command = new Command() ) .command("pull") .description("Pull any remote changes and apply them locally.") - .option( - "--fail-conflicts", - "Error on conflicts (both remote and local have changes on the same item)" - ) - .option( - "--raw", - "Push without using state, just overwrite. (Default, has no effect)" - ) .option("--yes", "Pull without needing confirmation") - .option( - "--stateful", - "Pull using state tracking (create .wmill folder and needed for --fail-conflicts)" - ) .option("--plain-secrets", "Pull secrets as plain text") .option("--json", "Use JSON instead of YAML") .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") .option("--skip-resources", "Skip syncing resources") + // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") @@ -1551,25 +1671,13 @@ const command = new Command() .action(pull as any) .command("push") .description("Push any local changes and apply them remotely.") - .option( - "--fail-conflicts", - "Error on conflicts (both remote and local have changes on the same item)" - ) - .option( - "--raw", - "Push without using state, just overwrite. (Default, has no effect)" - ) - .option( - "--stateful", - "Pull using state tracking (use .wmill folder and needed for --fail-conflicts)w" - ) - .option("--skip-pull", "(stateful only) Push without pulling first") .option("--yes", "Push without needing confirmation") .option("--plain-secrets", "Push secrets as plain text") .option("--json", "Use JSON instead of YAML") .option("--skip-variables", "Skip syncing variables (including secrets)") .option("--skip-secrets", "Skip syncing only secrets variables") .option("--skip-resources", "Skip syncing resources") + // .option("--skip-scripts-metadata", "Skip syncing scripts metadata, focus solely on logic") .option("--include-schedules", "Include syncing schedules") .option("--include-users", "Include syncing users") .option("--include-groups", "Include syncing groups") diff --git a/cli/types.ts b/cli/types.ts index 0d717c296e..0ba5a1c476 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -79,7 +79,8 @@ export function showDiff(local: string, remote: string) { log.info("Diff too large to display"); return; } - for (const part of Diff.diffLines(local, remote)) { + + for (const part of Diff.diffLines(local ?? '', remote ?? '')) { if (part.removed) { // print red if removed without newline finalString += `\x1b[31m${part.value}\x1b[0m`; @@ -157,8 +158,8 @@ export function parseFromPath(p: string, content: string): any { return p.endsWith(".yaml") ? yamlParseContent(p, content) : p.endsWith(".json") - ? JSON.parse(content) - : content; + ? JSON.parse(content) + : content; } export function parseFromFile(p: string): any { if (p.endsWith(".json")) { diff --git a/frontend/src/lib/hubPaths.json b/frontend/src/lib/hubPaths.json index 2adebb2b99..a9c887fe19 100644 --- a/frontend/src/lib/hubPaths.json +++ b/frontend/src/lib/hubPaths.json @@ -2,7 +2,8 @@ "gitSync_0": "hub/9087/sync-script-to-git-repo-windmill", "gitSync_1": "hub/9987/sync-script-to-git-repo-windmill", "gitSync_2": "hub/11498/sync-script-to-git-repo-windmill", - "gitSync": "hub/11533/sync-script-to-git-repo-windmill", + "gitSync_3": "hub/11533/sync-script-to-git-repo-windmill", + "gitSync": "hub/11580/sync-script-to-git-repo-windmill", "gitSyncTest_0": "hub/9073/git-repo-test-read-write-windmill", "gitSyncTest": "hub/11499/git-repo-test-read-write-windmill", "slackErrorHandler": "hub/9206/workspace-or-schedule-error-handler-slack",