feat(cli): split flows inline scripts

cli/flow.ts

@@ -1,14 +1,12 @@
 // deno-lint-ignore-file no-explicit-any
-import {
-  GlobalOptions,
-  isSuperset,
-  parseFromFile,
-  removeType,
-} from "./types.ts";
+import { GlobalOptions, isSuperset } from "./types.ts";
+import { parse as yamlParse } from "https://deno.land/std@0.184.0/yaml/mod.ts";
+
 import {
   colors,
   Command,
   Flow,
+  FlowModule,
   FlowService,
   JobService,
   Table,
@@ -23,16 +21,55 @@ export interface FlowFile {
   schema?: any;
 }
 
+let alreadySynced: string[] = [];
+
 export async function pushFlow(
   workspace: string,
   remotePath: string,
-  flow: Flow | FlowFile | undefined,
-  localFlow: FlowFile
+  localFlowPath: string,
+  workspaceId: string
 ): Promise<void> {
-  remotePath = removeType(remotePath, "flow");
+  if (alreadySynced.includes(localFlowPath)) {
+    return;
+  }
+  alreadySynced.push(localFlowPath);
+  let flow: Flow | undefined = undefined;
+  try {
+    flow = await FlowService.getFlowByPath({
+      workspace: workspaceId,
+      path: remotePath,
+    });
+  } catch {
+    // flow doesn't exist
+  }
+
+  if (!localFlowPath.endsWith("/")) {
+    localFlowPath += "/";
+  }
+  const localFlowRaw = await Deno.readTextFile(localFlowPath + "flow.yaml");
+  const localFlow = yamlParse(localFlowRaw) as FlowFile;
+
+  function extractInlineScripts(modules: FlowModule[]) {
+    return modules.flatMap((m) => {
+      if (m.value.type == "rawscript") {
+        const path = m.value.content.split(" ")[1];
+        m.value.content = Deno.readTextFileSync(localFlowPath + path);
+      } else if (m.value.type == "forloopflow") {
+        extractInlineScripts(m.value.modules);
+      } else if (m.value.type == "branchall") {
+        m.value.branches.forEach((b) => extractInlineScripts(b.modules));
+      } else if (m.value.type == "branchone") {
+        m.value.branches.forEach((b) => extractInlineScripts(b.modules));
+        extractInlineScripts(m.value.default);
+      }
+    });
+  }
+
+  extractInlineScripts(localFlow.value.modules);
 
   if (flow) {
     if (isSuperset(localFlow, flow)) {
+      console.log(colors.bold.green("Flow is up to date"));
       return;
     }
     await FlowService.updateFlow({
@@ -63,20 +100,12 @@ async function push(opts: Options, filePath: string, remotePath: string) {
   }
   const workspace = await resolveWorkspace(opts);
   await requireLogin(opts);
-  let flow: Flow | undefined = undefined;
-  try {
-    flow = await FlowService.getFlowByPath({
-      workspace: workspace.workspaceId,
-      path: remotePath,
-    });
-  } catch {
-    // flow doesn't exist
-  }
+
   await pushFlow(
     workspace.workspaceId,
     remotePath,
-    flow,
-    parseFromFile(filePath)
+    filePath,
+    workspace.workspaceId
   );
   console.log(colors.bold.underline.green("Flow pushed"));
 }

cli/script.ts

@@ -1,5 +1,5 @@
 // deno-lint-ignore-file no-explicit-any
-import { GlobalOptions, parseFromFile } from "./types.ts";
+import { GlobalOptions, parseFromFile, removeType } from "./types.ts";
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
 import {
   colors,
@@ -76,10 +76,11 @@ export async function handleFile(
   alreadySynced: string[]
 ): Promise<boolean> {
   if (
-    path.endsWith(".ts") ||
-    path.endsWith(".py") ||
-    path.endsWith(".go") ||
-    path.endsWith(".sh")
+    !path.includes(".inline_script.") &&
+    (path.endsWith(".ts") ||
+      path.endsWith(".py") ||
+      path.endsWith(".go") ||
+      path.endsWith(".sh"))
   ) {
     if (alreadySynced.includes(path)) {
       return true;
@@ -133,6 +134,9 @@ export async function handleFile(
           return true;
         }
       }
+      console.log(
+        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
+      );
       await ScriptService.createScript({
         workspace,
         requestBody: {
@@ -148,11 +152,10 @@ export async function handleFile(
           schema: typed?.schema,
         },
       });
-
-      console.log(
-        colors.yellow.bold(`Creating script with a parent ${remotePath}`)
-      );
     } else {
+      console.log(
+        colors.yellow.bold(`Creating script without parent ${remotePath}`)
+      );
       // no parent hash
       await ScriptService.createScript({
         workspace: workspace,
@@ -169,9 +172,6 @@ export async function handleFile(
           schema: typed?.schema,
         },
       });
-      console.log(
-        colors.yellow.bold(`Creating script without parent ${remotePath}`)
-      );
     }
     return true;
   }
@@ -243,6 +243,8 @@ export async function pushScript(
   workspace: string,
   remotePath: string
 ) {
+  remotePath = removeType(remotePath, "script");
+
   const data: ScriptFile | undefined = filePath
     ? parseFromFile(filePath)
     : undefined;
@@ -263,7 +265,7 @@ export async function pushScript(
     }
   }
 
-  console.log(colors.bold.yellow("Pushing script..."));
+  console.log(colors.bold.yellow(`Pushing script ${remotePath}...`));
   await ScriptService.createScript({
     workspace: workspace,
     requestBody: {

cli/sync.ts

@@ -13,6 +13,9 @@ import {
   VariableService,
   AppService,
   FlowService,
+  OpenFlow,
+  FlowModule,
+  RawScript,
 } from "./deps.ts";
 import {
   getTypeStrFromPath,
@@ -47,6 +50,7 @@ async function FSFSElement(p: string): Promise<DynFSElement> {
       isDirectory: isDir,
       path: localP.substring(p.length + 1),
       async *getChildren(): AsyncIterable<DynFSElement> {
+        if (!isDir) return [];
         for await (const e of Deno.readDir(localP)) {
           yield _internal_element(path.join(localP, e.name), e.isDirectory);
         }
@@ -74,14 +78,109 @@ function prioritizeName(name: string): string {
   return name;
 }
 
+const yamlOptions = {
+  sortKeys: (a: any, b: any) => {
+    return prioritizeName(a).localeCompare(prioritizeName(b));
+  },
+  noCompatMode: true,
+  noRefs: true,
+};
+
 function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
   function _internal_file(p: string, f: JSZip.JSZipObject): DynFSElement {
+    const isFlow = p.endsWith("flow.json");
+    function transformPath() {
+      if (isFlow) {
+        return p.replace("flow.json", "flow");
+      } else {
+        return useYaml && p.endsWith(".json")
+          ? p.replaceAll(".json", ".yaml")
+          : p;
+      }
+    }
+
+    interface InlineScript {
+      path: string;
+      content: string;
+    }
+    let counter = 0;
+    const seen_names = new Set<string>();
+    function assignPath(
+      summary: string | undefined,
+      language: RawScript.language
+    ): string {
+      let name;
+      if (summary && summary != "" && !seen_names.has(summary)) {
+        name = summary.toLowerCase().replaceAll(" ", "_");
+        seen_names.add(name);
+      } else {
+        name = `inline_script_${counter}`;
+        while (seen_names.has(name)) {
+          counter++;
+          name = `inline_script_${counter}`;
+        }
+        seen_names.add(name);
+      }
+      let ext;
+      if (language == "python3") ext = "py";
+      else if (language == "deno") ext = "ts";
+      else if (language == "go") ext = "go";
+      else if (language == "bash") ext = "sh";
+      return `${name}.inline_script.${ext}`;
+    }
+    function extractInlineScripts(modules: FlowModule[]): InlineScript[] {
+      return modules.flatMap((m) => {
+        if (m.value.type == "rawscript") {
+          const path = assignPath(m.summary, m.value.language);
+          const content = m.value.content;
+          m.value.content = "!inline " + path;
+          return [{ path: path, content: content }];
+        } else if (m.value.type == "forloopflow") {
+          return extractInlineScripts(m.value.modules);
+        } else if (m.value.type == "branchall") {
+          return m.value.branches.flatMap((b) =>
+            extractInlineScripts(b.modules)
+          );
+        } else if (m.value.type == "branchone") {
+          return [
+            ...m.value.branches.flatMap((b) => extractInlineScripts(b.modules)),
+            ...extractInlineScripts(m.value.default),
+          ];
+        } else {
+          return [];
+        }
+      });
+    }
+    const flowPath = transformPath();
     return {
-      isDirectory: false,
-      path: useYaml && p.endsWith(".json") ? p.replaceAll(".json", ".yaml") : p,
-      // deno-lint-ignore require-yield
+      isDirectory: isFlow,
+      path: flowPath,
       async *getChildren(): AsyncIterable<DynFSElement> {
-        throw new Error("Cannot get children of file");
+        if (isFlow) {
+          const flow: OpenFlow = JSON.parse(await f.async("text"));
+          const inlineScripts = extractInlineScripts(flow.value.modules);
+          for (const s of inlineScripts) {
+            yield {
+              isDirectory: false,
+              path: path.join(flowPath, s.path),
+              async *getChildren() {},
+              // deno-lint-ignore require-await
+              async getContentText() {
+                return s.content;
+              },
+            };
+          }
+
+          yield {
+            isDirectory: false,
+            path: path.join(flowPath, "flow.yaml"),
+            async *getChildren() {},
+            // deno-lint-ignore require-await
+            async getContentText() {
+              return yamlStringify(flow, yamlOptions);
+            },
+          };
+        }
       },
       // async getContentBytes(): Promise<Uint8Array> {
       //   return await f.async("uint8array");
@@ -89,13 +188,7 @@ function ZipFSElement(zip: JSZip, useYaml: boolean): DynFSElement {
       async getContentText(): Promise<string> {
         const content = await f.async("text");
         return useYaml && p.endsWith(".json")
-          ? yamlStringify(JSON.parse(content), {
-              sortKeys: (a, b) => {
-                return prioritizeName(a).localeCompare(prioritizeName(b));
-              },
-              noCompatMode: true,
-              noRefs: true,
-            })
+          ? yamlStringify(JSON.parse(content), yamlOptions)
           : content;
       },
     };
@@ -164,7 +257,6 @@ async function* readDirRecursiveWithIgnore(
   while (stack.length > 0) {
     const e = stack.pop()!;
     yield e;
-    if (!e.isDirectory) continue;
     for await (const e2 of e.c()) {
       stack.push({
         path: e2.path,

cli/types.ts

@@ -48,10 +48,14 @@ export function isSuperset(
   return Object.keys(subset).every((key) => {
     const eq = equal(subset[key], superset[key]);
     if (!eq) {
-      console.log(
-        `Found diff for ${key}:`,
-        showDiff(yamlStringify(subset[key]), yamlStringify(superset[key]))
-      );
+      const sub = subset[key];
+      const supers = superset[key];
+      if (!supers) {
+        console.log(`Key ${key} not found in remote`);
+      } else {
+        console.log(`Found diff for ${key}:`);
+        showDiff(yamlStringify(sub), yamlStringify(supers));
+      }
     }
     return eq;
   });
@@ -67,6 +71,13 @@ export function showDiff(local: string, remote: string) {
       // print green if added
       finalString += `\x1b[32m${part.value}\x1b[0m`;
     } else {
+      let lines = part.value.split("\n");
+
+      if (lines.length > 4) {
+        lines = lines.slice(0, 2);
+        lines.push("...");
+        lines = lines.concat(part.value.split("\n").slice(-2));
+      }
       // print white if unchanged
       finalString += `\x1b[37m${part.value}\x1b[0m`;
     }
@@ -99,7 +110,8 @@ export function pushObj(
   } else if (typeEnding === "variable") {
     pushVariable(workspace, p, befObj, newObj, plainSecrets);
   } else if (typeEnding === "flow") {
-    pushFlow(workspace, p, befObj, newObj);
+    const flowName = p.split(".flow/")[0];
+    pushFlow(workspace, flowName, flowName + ".flow", workspace);
   } else if (typeEnding === "resource") {
     pushResource(workspace, p, befObj, newObj);
   } else if (typeEnding === "resource-type") {
@@ -110,7 +122,11 @@ export function pushObj(
 }
 
 export function parseFromPath(p: string, content: string): any {
-  return p.endsWith(".yaml") ? yamlParse(content) : JSON.parse(content);
+  return p.endsWith(".yaml")
+    ? yamlParse(content)
+    : p.endsWith(".json")
+    ? JSON.parse(content)
+    : content;
 }
 export function parseFromFile(p: string): any {
   if (p.endsWith(".json")) {
@@ -131,6 +147,9 @@ export function getTypeStrFromPath(
   | "resource-type"
   | "folder"
   | "app" {
+  if (p.includes(".flow/")) {
+    return "flow";
+  }
   const parsed = path.parse(p);
   if (
     parsed.ext == ".go" ||
@@ -149,7 +168,6 @@ export function getTypeStrFromPath(
   if (
     typeEnding === "script" ||
     typeEnding === "variable" ||
-    typeEnding === "flow" ||
     typeEnding === "resource" ||
     typeEnding === "resource-type" ||
     typeEnding === "app"

cli/variable.ts

@@ -1,6 +1,11 @@
 // deno-lint-ignore-file no-explicit-any
 import { requireLogin, resolveWorkspace, validatePath } from "./context.ts";
-import { GlobalOptions, isSuperset, parseFromFile } from "./types.ts";
+import {
+  GlobalOptions,
+  isSuperset,
+  parseFromFile,
+  removeType,
+} from "./types.ts";
 import {
   colors,
   Command,
@@ -47,6 +52,8 @@ export async function pushVariable(
   localVariable: VariableFile,
   plainSecrets: boolean
 ): Promise<void> {
+  remotePath = removeType(remotePath, "variable");
+
   if (variable) {
     if (isSuperset(localVariable, variable)) {
       return;
@@ -61,7 +68,7 @@ export async function pushVariable(
       },
     });
   } else {
-    console.log(colors.yellow.bold("Creating new variable..."));
+    console.log(colors.yellow.bold(`Creating new variable ${remotePath}...`));
     await VariableService.createVariable({
       workspace,
       alreadyEncrypted: !plainSecrets,