feat(cli): Run flows & scripts (#940)

* Enable Script Running from CLI

* Improve Script Logic & Enable Flow run

* Update README

* Fix empty input 415 unsupported media type

* Add flow execution asciicast

* Allow reading inputs

* Add --silent & print result

* Updated syntax

* Update readme

* Fix superadmin users

* Handle values correctly

* Rework input parsing to try-catch JSON

* Accept all input types

* VHS scripts

* Test add Video to Markdown

* Use GIF only

* Final revisions

* I'm not sure why this works but stackoverflow told me
https://stackoverflow.com/questions/4279611/how-to-embed-a-video-into-github-readme-md/4279746#4279746

* Also rename file?

* Use MP4

* Use GIF

* Use MP4 again

* Revert "Use MP4 again"

This reverts commit d3ed4dc28a.

cli/README.md

@@ -1,8 +1,7 @@
 # Windmill CLI
 
 A simple CLI allowing interactions with windmill from the command line.
-
-[![asciicast](https://asciinema.org/a/533968.svg)](https://asciinema.org/a/533968)
+![](./vhs/output/setup.gif)
 
 ## Installation
 
@@ -14,9 +13,16 @@ Update to the latest version using `wmill upgrade`.
 
 Setup the CLI by running `wmill setup`. This will guide you through the setup process easily.
 
-## Pushing Resources
+## Running Flows & Scripts
 
-The CLI can push resource specifications to a windmill instance. See the [examples/](./examples/) folder for formats.
+Run a script or flow using `wmill flow/script run u/username/path/to/script` and pass any inputs using `--input/-i <name>=<value>` curl-style syntax using `-i @-` for stdin or `-i @<filename>` is also supported
+
+Flow Steps and Logs will be streamed during execution automatically.
+![](./vhs/output/run-flow.gif)
+
+## Pushing Resources, Scripts & More
+
+The CLI can push specifications to a windmill instance. See the [examples/](./examples/) folder for formats.
 
 ### Pushing a folder
 

cli/flow.ts

@@ -1,5 +1,8 @@
 import { Command } from "https://deno.land/x/cliffy@v0.25.4/command/command.ts";
-import { FlowService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
+import {
+  FlowService,
+  JobService,
+} from "https://deno.land/x/windmill@v1.50.0/mod.ts";
 import { GlobalOptions } from "./types.ts";
 import {
   Flow,
@@ -8,6 +11,7 @@ import {
 import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
 import { getContext } from "./context.ts";
 import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
+import { resolve, track_job } from "./script.ts";
 
 type Options = GlobalOptions;
 
@@ -100,6 +104,52 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
     )
     .render();
 }
+async function run(
+  opts: GlobalOptions & {
+    input: string[];
+    silent: boolean;
+  },
+  path: string
+) {
+  const { workspace } = await getContext(opts);
+
+  const input = await resolve(opts.input);
+
+  const id = await JobService.runFlowByPath({
+    workspace,
+    path,
+    requestBody: input,
+  });
+
+  let i = 0;
+  while (true) {
+    const jobInfo = await JobService.getJob({ workspace, id });
+    if (jobInfo.flow_status!.modules.length <= i) {
+      break;
+    }
+    const module = jobInfo.flow_status!.modules[i];
+
+    if (module.job) {
+      if (!opts.silent) {
+        console.log("====== Job " + (i + 1) + " ======");
+        await track_job(workspace, module.job);
+      }
+    } else {
+      console.log(module.type);
+      await new Promise((resolve, _) =>
+        setTimeout(() => resolve(undefined), 100)
+      );
+      continue;
+    }
+    i++;
+  }
+
+  if (!opts.silent) {
+    console.log(colors.green.underline.bold("Flow ran to completion"));
+  }
+  const jobInfo = await JobService.getCompletedJob({ workspace, id });
+  console.log(jobInfo.result ?? {});
+}
 
 const command = new Command()
   .description("flow related commands")
@@ -110,6 +160,17 @@ const command = new Command()
     "push a local flow spec. This overrides any remote versions."
   )
   .arguments("<file_path:string> <remote_path:string>")
-  .action(push as any);
+  .action(push as any)
+  .command("run", "run a flow by path.")
+  .arguments("<path:string>")
+  .option(
+    "-i --input [inputs...:string]",
+    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones."
+  )
+  .option(
+    "-s --silent",
+    "Do not ouput anything other then the final output. Useful for scripting."
+  )
+  .action(run as any);
 
 export default command;

cli/script.ts

@@ -3,8 +3,12 @@ import { ScriptService } from "https://deno.land/x/windmill@v1.50.0/mod.ts";
 import { GlobalOptions } from "./types.ts";
 import { colors } from "https://deno.land/x/cliffy@v0.25.4/ansi/colors.ts";
 import { getContext } from "./context.ts";
-import { Script } from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
+import {
+  JobService,
+  Script,
+} from "https://deno.land/x/windmill@v1.50.0/windmill-api/index.ts";
 import { Table } from "https://deno.land/x/cliffy@v0.25.4/table/table.ts";
+import { readAll } from "https://deno.land/std@0.165.0/streams/mod.ts";
 
 type ScriptFile = {
   parent_hash?: string;
@@ -168,6 +172,158 @@ async function list(opts: GlobalOptions & { showArchived?: boolean }) {
     .render();
 }
 
+export async function resolve(inputs: string[]): Promise<Record<string, any>> {
+  let result = {};
+
+  if (!inputs) {
+    return result;
+  }
+
+  for (const input of inputs) {
+    let data: string;
+    if (input.startsWith("@")) {
+      if (input == "@-") {
+        data = new TextDecoder().decode(await readAll(Deno.stdin));
+      } else {
+        data = await Deno.readTextFile(input.substring(1));
+      }
+    } else {
+      if (input.startsWith("{")) {
+        data = input;
+      } else {
+        const key = input.split("=", 1)[0];
+        const value = input.substring(key.length + 1);
+        let o;
+        try {
+          o = JSON.parse(value);
+        } catch {
+          o = value;
+        }
+        data = JSON.stringify(Object.fromEntries([[key, o]]));
+      }
+    }
+    let jsonObj;
+    try {
+      jsonObj = JSON.parse(data);
+    } catch {
+      jsonObj = data;
+    }
+    result = { ...result, ...jsonObj };
+  }
+  return result;
+}
+
+async function run(
+  opts: GlobalOptions & {
+    input: string[];
+    silent: boolean;
+  },
+  path: string
+) {
+  const { workspace } = await getContext(opts);
+
+  const input = await resolve(opts.input);
+  const id = await JobService.runScriptByPath({
+    workspace,
+    path,
+    requestBody: input,
+  });
+
+  if (!opts.silent) {
+    await track_job(workspace, id);
+  }
+
+  while (true) {
+    try {
+      const result =
+        (await JobService.getCompletedJob({ workspace, id })).result ?? {};
+      console.log(result);
+
+      break;
+    } catch {
+      new Promise((resolve, _) => setTimeout(() => resolve(undefined), 100));
+    }
+  }
+}
+
+export async function track_job(workspace: string, id: string) {
+  try {
+    const result = await JobService.getCompletedJob({ workspace, id });
+
+    console.log(result.logs);
+    console.log(colors.bold.underline.green("Job Completed"));
+    return;
+  } catch {
+    /* ignore */
+  }
+
+  console.log(colors.yellow("Waiting for Job " + id + " to start..."));
+
+  let logOffset = 0;
+  let running = false;
+  let retry = 0;
+  while (true) {
+    let updates: {
+      running?: boolean | undefined;
+      completed?: boolean | undefined;
+      new_logs?: string | undefined;
+    };
+    try {
+      updates = await JobService.getJobUpdates({
+        workspace,
+        id,
+        logOffset,
+        running,
+      });
+    } catch {
+      retry++;
+      if (retry > 3) {
+        console.log("failed to get job updated. skipping log streaming.");
+        break;
+      }
+      continue;
+    }
+
+    if (!running && updates.running === true) {
+      running = true;
+      console.log(colors.green("Job running. Streaming logs..."));
+    }
+
+    if (updates.new_logs) {
+      console.log(updates.new_logs);
+      logOffset += updates.new_logs.length;
+    }
+
+    if (updates.completed === true) {
+      running = false;
+      break;
+    }
+
+    if (running && updates.running === false) {
+      running = false;
+      console.log(
+        colors.yellow("Job suspended. Waiting for it to continue...")
+      );
+    }
+  }
+  await new Promise((resolve, _) => setTimeout(() => resolve(undefined), 1000));
+
+  try {
+    const final_job = await JobService.getCompletedJob({ workspace, id });
+    if ((final_job.logs?.length ?? -1) > logOffset) {
+      console.log(final_job.logs!.substring(logOffset));
+    }
+
+    if (final_job.success) {
+      console.log(colors.bold.underline.green("Job Completed"));
+    } else {
+      console.log(colors.bold.underline.red("Job Completed"));
+    }
+  } catch {
+    console.log("Job appears to have completed, but no data can be retrieved");
+  }
+}
+
 async function show(opts: GlobalOptions, path: string) {
   const { workspace } = await getContext(opts);
   const s = await ScriptService.getScriptByPath({ workspace, path });
@@ -189,6 +345,17 @@ const command = new Command()
   .action(push as any)
   .command("show", "show a scripts content")
   .arguments("<path:string>")
-  .action(show as any);
+  .action(show as any)
+  .command("run", "run a script by path")
+  .arguments("<path:string>")
+  .option(
+    "-i --input [inputs...:string]",
+    "Inputs specified as JSON objects or simply as <name>=<value>. Supports file inputs using @<filename> and stdin using @- these also need to be formatted as JSON. Later inputs override earlier ones."
+  )
+  .option(
+    "-s --silent",
+    "Do not ouput anything other then the final output. Useful for scripting."
+  )
+  .action(run as any);
 
 export default command;

cli/user.ts

@@ -42,7 +42,7 @@ async function list(opts: GlobalOptions) {
 
 async function add(
   opts: GlobalOptions & {
-    superAdmin?: boolean;
+    superadmin?: boolean;
     company?: string;
     name?: string;
   },
@@ -55,7 +55,7 @@ async function add(
     requestBody: {
       email,
       password: password_final,
-      super_admin: opts.superAdmin ?? false,
+      super_admin: opts.superadmin ?? false,
       company: opts.company,
       name: opts.name,
     },

cli/vhs/README.md

@@ -0,0 +1,3 @@
+# VHS Files
+
+This folder includes [vhs](https://github.com/charmbracelet/vhs) files for demo purposes.

cli/vhs/example.flow.json

@@ -0,0 +1,55 @@
+{
+  "summary": "",
+  "description": "",
+  "value": {
+    "modules": [
+      {
+        "id": "a",
+        "value": {
+          "lock": "",
+          "path": null,
+          "type": "rawscript",
+          "content": "// import * as wmill from \"https://deno.land/x/windmill@v1.50.0/mod.ts\"\n\nexport async function main(x: string) {\n  console.log(\"Hello from Deno! The argument x is \" + x);\n  return x;\n}\n",
+          "language": "deno",
+          "input_transforms": {
+            "x": {
+              "expr": "`${flow_input.x}`",
+              "type": "javascript"
+            }
+          }
+        },
+        "summary": null,
+        "stop_after_if": null,
+        "input_transforms": {}
+      },
+      {
+        "id": "b",
+        "value": {
+          "lock": "",
+          "path": null,
+          "type": "rawscript",
+          "content": "def main():\n  print(\"Hello from Step 2, in Python!\");\n",
+          "language": "python3",
+          "input_transforms": {}
+        },
+        "summary": null,
+        "stop_after_if": null,
+        "input_transforms": {}
+      }
+    ],
+    "failure_module": null
+  },
+  "schema": {
+    "type": "object",
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "required": [],
+    "properties": {
+      "x": {
+        "type": "string",
+        "format": "",
+        "default": "",
+        "description": ""
+      }
+    }
+  }
+}

cli/vhs/run-flow.tape

@@ -0,0 +1,68 @@
+# VHS documentation
+#
+# Output:
+#   Output <path>.gif               Create a GIF output at the given <path>
+#   Output <path>.mp4               Create an MP4 output at the given <path>
+#   Output <path>.webm              Create a WebM output at the given <path>
+#
+# Require:
+#   Require <string>                Ensure a program is on the $PATH to proceed
+#
+# Settings:
+#   Set FontSize <number>           Set the font size of the terminal
+#   Set FontFamily <string>         Set the font family of the terminal
+#   Set Height <number>             Set the height of the terminal
+#   Set Width <number>              Set the width of the terminal
+#   Set LetterSpacing <float>       Set the font letter spacing (tracking)
+#   Set LineHeight <float>          Set the font line height
+#   Set LoopOffset <float>%         Set the starting frame offset for the GIF loop
+#   Set Theme <json|string>         Set the theme of the terminal
+#   Set Padding <number>            Set the padding of the terminal
+#   Set Framerate <number>          Set the framerate of the recording
+#   Set PlaybackSpeed <float>       Set the playback speed of the recording
+#
+# Sleep:
+#   Sleep <time>                    Sleep for a set amount of <time> in seconds
+#
+# Type:
+#   Type[@<time>] "<characters>"    Type <characters> into the terminal with a
+#                                   <time> delay between each character
+#
+# Keys:
+#   Backspace[@<time>] [number]     Press the Backspace key
+#   Down[@<time>] [number]          Press the Down key
+#   Enter[@<time>] [number]         Press the Enter key
+#   Space[@<time>] [number]         Press the Space key
+#   Tab[@<time>] [number]           Press the Tab key
+#   Left[@<time>] [number]          Press the Left Arrow key
+#   Right[@<time>] [number]         Press the Right Arrow key
+#   Up[@<time>] [number]            Press the Up Arrow key
+#   Down[@<time>] [number]          Press the Down Arrow key
+#   Ctrl+<key>                      Press the Control key + <key> (e.g. Ctrl+C)
+#
+# Display:
+#   Hide                            Hide the subsequent commands from the output
+#   Show                            Show the subsequent commands in the output
+
+Output output/run-flow.mp4
+Output output/run-flow.gif
+
+Set Theme nord
+
+Require wmill
+
+Set FontSize 16
+Set Width 1200
+Set Height 600
+
+Hide
+
+Type "name=$(cat /proc/sys/kernel/random/uuid | sed 's/[-]//g' | head -c 20; echo;)" Enter
+Type "cat ./example.flow.json | wmill flow push /dev/stdin u/admin/$name" Enter
+Sleep 1s
+Ctrl+L
+
+Show
+
+Type "wmill flow run u/admin/$name -i x=5" Enter
+Sleep 10s

cli/vhs/setup.tape

@@ -0,0 +1,99 @@
+# VHS documentation
+#
+# Output:
+#   Output <path>.gif               Create a GIF output at the given <path>
+#   Output <path>.mp4               Create an MP4 output at the given <path>
+#   Output <path>.webm              Create a WebM output at the given <path>
+#
+# Require:
+#   Require <string>                Ensure a program is on the $PATH to proceed
+#
+# Settings:
+#   Set FontSize <number>           Set the font size of the terminal
+#   Set FontFamily <string>         Set the font family of the terminal
+#   Set Height <number>             Set the height of the terminal
+#   Set Width <number>              Set the width of the terminal
+#   Set LetterSpacing <float>       Set the font letter spacing (tracking)
+#   Set LineHeight <float>          Set the font line height
+#   Set LoopOffset <float>%         Set the starting frame offset for the GIF loop
+#   Set Theme <json|string>         Set the theme of the terminal
+#   Set Padding <number>            Set the padding of the terminal
+#   Set Framerate <number>          Set the framerate of the recording
+#   Set PlaybackSpeed <float>       Set the playback speed of the recording
+#
+# Sleep:
+#   Sleep <time>                    Sleep for a set amount of <time> in seconds
+#
+# Type:
+#   Type[@<time>] "<characters>"    Type <characters> into the terminal with a
+#                                   <time> delay between each character
+#
+# Keys:
+#   Backspace[@<time>] [number]     Press the Backspace key
+#   Down[@<time>] [number]          Press the Down key
+#   Enter[@<time>] [number]         Press the Enter key
+#   Space[@<time>] [number]         Press the Space key
+#   Tab[@<time>] [number]           Press the Tab key
+#   Left[@<time>] [number]          Press the Left Arrow key
+#   Right[@<time>] [number]         Press the Right Arrow key
+#   Up[@<time>] [number]            Press the Up Arrow key
+#   Down[@<time>] [number]          Press the Down Arrow key
+#   Ctrl+<key>                      Press the Control key + <key> (e.g. Ctrl+C)
+#
+# Display:
+#   Hide                            Hide the subsequent commands from the output
+#   Show                            Show the subsequent commands in the output
+
+Output output/setup.mp4
+Output output/setup.gif
+
+Set Theme nord
+
+Require wmill
+Require docker
+
+Set FontSize 16
+Set Width 1200
+Set Height 600
+
+Hide
+
+Type "rm -r ~/.config/windmill" Enter
+Type "cd ../../" Enter
+# Type "docker compose pull" Enter
+# Type "docker compose down" Enter
+Ctrl+L
+
+Show
+
+# once wait is supported see https://github.com/charmbracelet/vhs/issues/70
+# Type "docker compose up -d" Enter Sleep 1400ms
+
+Type "wmill setup" Enter Sleep 500ms
+Type "http://localhost" Enter Sleep 500ms
+Type "local" Enter Sleep 500ms
+Enter # Use Username/Password Sleep 500ms
+Type "admin@windmill.dev" Enter Sleep 500ms
+Type "changeme" Enter Sleep 500ms
+Down Enter Sleep 500ms
+
+Sleep 1s
+
+Type "wmill user" Enter Sleep 500ms
+Type "wmill user add admin@example.com password12 --superadmin" Enter Sleep 500ms
+Type "wmill user" Enter Sleep 500ms
+
+Sleep 1s
+
+Type "wmill login" Enter Sleep 500ms
+Type "admin@example.com" Enter Sleep 500ms
+Type "password12" Enter Sleep 500ms
+
+Sleep 1s
+
+Type "wmill user" Enter Sleep 500ms
+Type "wmill user remove admin@windmill.dev" Enter Sleep 500ms
+
+Type "wmill user" Enter Sleep 500ms
+
+Sleep 10s