From 93ae0d71a4cb807854a4cefa0dfa2e5bbf5249b9 Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 5 May 2023 23:25:57 +0200 Subject: [PATCH] feat(cli): split flows inline scripts --- cli/flow.ts | 69 +++++++++++++++++++--------- cli/script.ts | 28 ++++++------ cli/sync.ts | 116 +++++++++++++++++++++++++++++++++++++++++++----- cli/types.ts | 32 ++++++++++--- cli/variable.ts | 11 ++++- 5 files changed, 202 insertions(+), 54 deletions(-) diff --git a/cli/flow.ts b/cli/flow.ts index 1b802a518f..0f2bbac22d 100644 --- a/cli/flow.ts +++ b/cli/flow.ts @@ -1,14 +1,12 @@ // deno-lint-ignore-file no-explicit-any -import { - GlobalOptions, - isSuperset, - parseFromFile, - removeType, -} from "./types.ts"; +import { GlobalOptions, isSuperset } from "./types.ts"; +import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts"; + import { colors, Command, Flow, + FlowModule, FlowService, JobService, Table, @@ -23,16 +21,55 @@ export interface FlowFile { schema?: any; } +let alreadySynced: string[] = []; + export async function pushFlow( workspace: string, remotePath: string, - flow: Flow | FlowFile | undefined, - localFlow: FlowFile + localFlowPath: string, + workspaceId: string ): Promise { - remotePath = removeType(remotePath, "flow"); + if (alreadySynced.includes(localFlowPath)) { + return; + } + alreadySynced.push(localFlowPath); + let flow: Flow | undefined = undefined; + try { + flow = await FlowService.getFlowByPath({ + workspace: workspaceId, + path: remotePath, + }); + } catch { + // flow doesn't exist + } + + if (!localFlowPath.endsWith("/")) { + localFlowPath += "/"; + } + const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml"); + const localFlow = yamlParse(localFlowRaw) as FlowFile; + + function extractInlineScripts(modules: FlowModule[]) { + return modules.flatMap((m) => { + if (m.value.type == "rawscript") { + const path = m.value.content.split(" ")[1]; + m.value.content = Deno.readTextFileSync(localFlowPath + path); + } else if (m.value.type == "forloopflow") { + extractInlineScripts(m.value.modules); + } else if (m.value.type == "branchall") { + m.value.branches.forEach((b) => extractInlineScripts(b.modules)); + } else if (m.value.type == "branchone") { + m.value.branches.forEach((b) => extractInlineScripts(b.modules)); + extractInlineScripts(m.value.default); + } + }); + } + + extractInlineScripts(localFlow.value.modules); if (flow) { if (isSuperset(localFlow, flow)) { + console.log(colors.bold.green("Flow is up to date")); return; } await FlowService.updateFlow({ @@ -63,20 +100,12 @@ async function push(opts: Options, filePath: string, remotePath: string) { } const workspace = await resolveWorkspace(opts); await requireLogin(opts); - let flow: Flow | undefined = undefined; - try { - flow = await FlowService.getFlowByPath({ - workspace: workspace.workspaceId, - path: remotePath, - }); - } catch { - // flow doesn't exist - } + await pushFlow( workspace.workspaceId, remotePath, - flow, - parseFromFile(filePath) + filePath, + workspace.workspaceId ); console.log(colors.bold.underline.green("Flow pushed")); } diff --git a/cli/script.ts b/cli/script.ts index f6b2acbac2..6b0c45e0a1 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -1,5 +1,5 @@ // deno-lint-ignore-file no-explicit-any -import { GlobalOptions, parseFromFile } from "./types.ts"; +import { GlobalOptions, parseFromFile, removeType } from "./types.ts"; import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; import { colors, @@ -76,10 +76,11 @@ export async function handleFile( alreadySynced: string[] ): Promise { if ( - path.endsWith(".ts") || - path.endsWith(".py") || - path.endsWith(".go") || - path.endsWith(".sh") + !path.includes(".inline_script.") && + (path.endsWith(".ts") || + path.endsWith(".py") || + path.endsWith(".go") || + path.endsWith(".sh")) ) { if (alreadySynced.includes(path)) { return true; @@ -133,6 +134,9 @@ export async function handleFile( return true; } } + console.log( + colors.yellow.bold(`Creating script with a parent ${remotePath}`) + ); await ScriptService.createScript({ workspace, requestBody: { @@ -148,11 +152,10 @@ export async function handleFile( schema: typed?.schema, }, }); - - console.log( - colors.yellow.bold(`Creating script with a parent ${remotePath}`) - ); } else { + console.log( + colors.yellow.bold(`Creating script without parent ${remotePath}`) + ); // no parent hash await ScriptService.createScript({ workspace: workspace, @@ -169,9 +172,6 @@ export async function handleFile( schema: typed?.schema, }, }); - console.log( - colors.yellow.bold(`Creating script without parent ${remotePath}`) - ); } return true; } @@ -243,6 +243,8 @@ export async function pushScript( workspace: string, remotePath: string ) { + remotePath = removeType(remotePath, "script"); + const data: ScriptFile | undefined = filePath ? parseFromFile(filePath) : undefined; @@ -263,7 +265,7 @@ export async function pushScript( } } - console.log(colors.bold.yellow("Pushing script...")); + console.log(colors.bold.yellow(`Pushing script ${remotePath}...`)); await ScriptService.createScript({ workspace: workspace, requestBody: { diff --git a/cli/sync.ts b/cli/sync.ts index 96ab35e1d2..3dc7b9b27f 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -13,6 +13,9 @@ import { VariableService, AppService, FlowService, + OpenFlow, + FlowModule, + RawScript, } from "./deps.ts"; import { getTypeStrFromPath, @@ -47,6 +50,7 @@ async function FSFSElement(p: string): Promise { isDirectory: isDir, path: localP.substring(p.length + 1), async *getChildren(): AsyncIterable { + if (!isDir) return []; for await (const e of Deno.readDir(localP)) { yield _internal_element(path.join(localP, e.name), e.isDirectory); } @@ -74,14 +78,109 @@ function prioritizeName(name: string): string { return name; } +const yamlOptions = { + sortKeys: (a: any, b: any) => { + return prioritizeName(a).localeCompare(prioritizeName(b)); + }, + noCompatMode: true, + noRefs: true, +}; + function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement { + const isFlow = p.endsWith("flow.json"); + function transformPath() { + if (isFlow) { + return p.replace("flow.json", "flow"); + } else { + return useYaml && p.endsWith(".json") + ? p.replaceAll(".json", ".yaml") + : p; + } + } + + interface InlineScript { + path: string; + content: string; + } + let counter = 0; + const seen_names = new Set(); + function assignPath( + summary: string | undefined, + language: RawScript.language + ): string { + let name; + if (summary && summary != "" && !seen_names.has(summary)) { + name = summary.toLowerCase().replaceAll(" ", "_"); + seen_names.add(name); + } else { + name = `inline_script_${counter}`; + while (seen_names.has(name)) { + counter++; + name = `inline_script_${counter}`; + } + seen_names.add(name); + } + let ext; + if (language == "python3") ext = "py"; + else if (language == "deno") ext = "ts"; + else if (language == "go") ext = "go"; + else if (language == "bash") ext = "sh"; + return `${name}.inline_script.${ext}`; + } + function extractInlineScripts(modules: FlowModule[]): InlineScript[] { + return modules.flatMap((m) => { + if (m.value.type == "rawscript") { + const path = assignPath(m.summary, m.value.language); + const content = m.value.content; + m.value.content = "!inline " + path; + return [{ path: path, content: content }]; + } else if (m.value.type == "forloopflow") { + return extractInlineScripts(m.value.modules); + } else if (m.value.type == "branchall") { + return m.value.branches.flatMap((b) => + extractInlineScripts(b.modules) + ); + } else if (m.value.type == "branchone") { + return [ + ...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)), + ...extractInlineScripts(m.value.default), + ]; + } else { + return []; + } + }); + } + const flowPath = transformPath(); return { - isDirectory: false, - path: useYaml && p.endsWith(".json") ? p.replaceAll(".json", ".yaml") : p, - // deno-lint-ignore require-yield + isDirectory: isFlow, + path: flowPath, async *getChildren(): AsyncIterable { - throw new Error("Cannot get children of file"); + if (isFlow) { + const flow: OpenFlow = JSON.parse(await f.async("text")); + const inlineScripts = extractInlineScripts(flow.value.modules); + for (const s of inlineScripts) { + yield { + isDirectory: false, + path: path.join(flowPath, s.path), + async *getChildren() {}, + // deno-lint-ignore require-await + async getContentText() { + return s.content; + }, + }; + } + + yield { + isDirectory: false, + path: path.join(flowPath, "flow.yaml"), + async *getChildren() {}, + // deno-lint-ignore require-await + async getContentText() { + return yamlStringify(flow, yamlOptions); + }, + }; + } }, // async getContentBytes(): Promise { // return await f.async("uint8array"); @@ -89,13 +188,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement { async getContentText(): Promise { const content = await f.async("text"); return useYaml && p.endsWith(".json") - ? yamlStringify(JSON.parse(content), { - sortKeys: (a, b) => { - return prioritizeName(a).localeCompare(prioritizeName(b)); - }, - noCompatMode: true, - noRefs: true, - }) + ? yamlStringify(JSON.parse(content), yamlOptions) : content; }, }; @@ -164,7 +257,6 @@ async function* readDirRecursiveWithIgnore( while (stack.length > 0) { const e = stack.pop()!; yield e; - if (!e.isDirectory) continue; for await (const e2 of e.c()) { stack.push({ path: e2.path, diff --git a/cli/types.ts b/cli/types.ts index 6c34c2eaf8..f15337f462 100644 --- a/cli/types.ts +++ b/cli/types.ts @@ -48,10 +48,14 @@ export function isSuperset( return Object.keys(subset).every((key) => { const eq = equal(subset[key], superset[key]); if (!eq) { - console.log( - `Found diff for ${key}:`, - showDiff(yamlStringify(subset[key]), yamlStringify(superset[key])) - ); + const sub = subset[key]; + const supers = superset[key]; + if (!supers) { + console.log(`Key ${key} not found in remote`); + } else { + console.log(`Found diff for ${key}:`); + showDiff(yamlStringify(sub), yamlStringify(supers)); + } } return eq; }); @@ -67,6 +71,13 @@ export function showDiff(local: string, remote: string) { // print green if added finalString += `\x1b[32m${part.value}\x1b[0m`; } else { + let lines = part.value.split("\n"); + + if (lines.length > 4) { + lines = lines.slice(0, 2); + lines.push("..."); + lines = lines.concat(part.value.split("\n").slice(-2)); + } // print white if unchanged finalString += `\x1b[37m${part.value}\x1b[0m`; } @@ -99,7 +110,8 @@ export function pushObj( } else if (typeEnding === "variable") { pushVariable(workspace, p, befObj, newObj, plainSecrets); } else if (typeEnding === "flow") { - pushFlow(workspace, p, befObj, newObj); + const flowName = p.split(".flow/")[0]; + pushFlow(workspace, flowName, flowName + ".flow", workspace); } else if (typeEnding === "resource") { pushResource(workspace, p, befObj, newObj); } else if (typeEnding === "resource-type") { @@ -110,7 +122,11 @@ export function pushObj( } export function parseFromPath(p: string, content: string): any { - return p.endsWith(".yaml") ? yamlParse(content) : JSON.parse(content); + return p.endsWith(".yaml") + ? yamlParse(content) + : p.endsWith(".json") + ? JSON.parse(content) + : content; } export function parseFromFile(p: string): any { if (p.endsWith(".json")) { @@ -131,6 +147,9 @@ export function getTypeStrFromPath( | "resource-type" | "folder" | "app" { + if (p.includes(".flow/")) { + return "flow"; + } const parsed = path.parse(p); if ( parsed.ext == ".go" || @@ -149,7 +168,6 @@ export function getTypeStrFromPath( if ( typeEnding === "script" || typeEnding === "variable" || - typeEnding === "flow" || typeEnding === "resource" || typeEnding === "resource-type" || typeEnding === "app" diff --git a/cli/variable.ts b/cli/variable.ts index aa1d0cb6f3..59b2226c60 100644 --- a/cli/variable.ts +++ b/cli/variable.ts @@ -1,6 +1,11 @@ // deno-lint-ignore-file no-explicit-any import { requireLogin, resolveWorkspace, validatePath } from "./context.ts"; -import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts"; +import { + GlobalOptions, + isSuperset, + parseFromFile, + removeType, +} from "./types.ts"; import { colors, Command, @@ -47,6 +52,8 @@ export async function pushVariable( localVariable: VariableFile, plainSecrets: boolean ): Promise { + remotePath = removeType(remotePath, "variable"); + if (variable) { if (isSuperset(localVariable, variable)) { return; @@ -61,7 +68,7 @@ export async function pushVariable( }, }); } else { - console.log(colors.yellow.bold("Creating new variable...")); + console.log(colors.yellow.bold(`Creating new variable ${remotePath}...`)); await VariableService.createVariable({ workspace, alreadyEncrypted: !plainSecrets,