From 161f793ae6a67761709d4ced2de060c9546b2d3b Mon Sep 17 00:00:00 2001 From: Ruben Fiszel Date: Fri, 23 Jun 2023 15:08:33 +0200 Subject: [PATCH] feat: release wmillbench publicly --- .github/change-versions.sh | 1 + benchmarks/README.md | 87 ++++++++++++++++++++++++-------------- benchmarks/main.ts | 35 +++++++++++++-- benchmarks/worker.ts | 51 ++++++++++++++++++++-- 4 files changed, 136 insertions(+), 38 deletions(-) diff --git a/.github/change-versions.sh b/.github/change-versions.sh index 9e230bbf4f..0810a2029e 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -5,6 +5,7 @@ echo "Updating versions to: $VERSION" sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts +sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json diff --git a/benchmarks/README.md b/benchmarks/README.md index 9500c3defe..16470112f2 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,47 +1,72 @@ # Benchmarks -This folder includes a small deno/ts utility to benchmark execution of jobs & flows. +This folder includes a small deno/ts utility to benchmark execution of jobs & +flows. + +## Installation + +Install the `wmill` CLI tool using +`deno install --unstable -A https://deno.land/x/wmillbench/main.ts`. + +Update to the latest version using `wmillbench upgrade`. + +## Quickstart + +Have your instance expose prometheus metrics (METRICS_ADDR=1). + +Then ``` -Usage: windmill-bench +wmillbench -s 1 -e admin@windmill.dev -p changeme --host YOUR_HOST +``` + +## Usage + +``` + +Usage: wmillbench Description: - Run Benchmark to measure throughput of windmill. +Run Benchmark to measure throughput of windmill. Options: - -h, --help - Show this help. - -V, --version - Show the version number for this program. - --host - The windmill host to benchmark. (Default: "http://127.0.0.1/") - --workers - The number of workers to run at once. (Default: 1) - -s, --seconds - How long to run the benchmark for (in seconds). (Default: 30) - -e, --email - The email to use to login. - -p, --password - The password to use to login. - -t, --token - The token to use when talking to the API server. Preferred over manual login. - -w, --workspace - The workspace to spawn scripts from. (Default: "starter") - -m, --metrics - The url to scrape metrics from. (Default: "http://localhost:8001/metrics") - --export-json - If set, exports will be into a JSON file. - --export-csv - If set, exports will be into a csv file. - --export-histograms [histograms...] - Mark metrics (without label) that are reported as histograms to export. - --export-simple [simple...] - Mark metrics (without label) that are reported as simple values. - --maximum-throughput - Maximum number of jobs/flows to start in one second. (Default: Infinity) - --use-flows - Run flows instead of jobs. - --histogram-buckets [buckets...] - Define what buckets to collect from histograms. (Default: [ - "+Inf", "10", "5", - "2.5", "2.5", "1", - "0.5", "0.25", "0.1", - "0.05", "0.025", "0.01", - "0.005" - ]) +-h, --help - Show this help. -V, --version - Show the version number for this +program. --host - The windmill host to benchmark. (Default: +"http://127.0.0.1:8000/") --workers - The number of workers to run at +once. (Default: 1) -s, --seconds - How long to run the benchmark for +(in seconds). (Default: 30) -e, --email - The email to use to login. -p, +--password - The password to use to login. -t, --token - The +token to use when talking to the API server. Preferred over manual login. -w, +--workspace - The workspace to spawn scripts from. (Default: +"starter") -m, --metrics - The url to scrape metrics from. (Default: +"http://localhost:8001/metrics") --export-json - If set, exports +will be into a JSON file. --export-csv - If set, exports will be +into a csv file. --export-histograms [histograms...] - Mark metrics (without +label) that are reported as histograms to export. --export-simple [simple...] - +Mark metrics (without label) that are reported as simple values. +--maximum-throughput - Maximum number of jobs/flows to +start in one second. (Default: Infinity) --use-flows - Run flows instead of +jobs. --histogram-buckets [buckets...] - Define what buckets to collect from +histograms. (Default: [ "+Inf", "10", "5", "2.5", "2.5", "1", "0.5", "0.25", +"0.1", "0.05", "0.025", "0.01", "0.005" ]) Environment variables: - WM_TOKEN - The token to use when talking to the API server. Preferred over manual login. - WM_WORKSPACE - The workspace to spawn scripts from. +WM_TOKEN - The token to use when talking to the API server. Preferred +over manual login. WM_WORKSPACE - The workspace to spawn scripts +from. + ``` -This will run a simple benchmark against localhost (the default admin email + password are set above), all execution is done in the "bench" workspace (as set via `--workspace`). +This will run a simple benchmark against localhost (the default admin email + +password are set above), all execution is done in the "bench" workspace (as set +via `--workspace`). -Metrics are exported to JSON will only include mean & stdev, histograms get one entry for each bucket. -CSV will include a full list of all values scraped. +Metrics are exported to JSON will only include mean & stdev, histograms get one +entry for each bucket. CSV will include a full list of all values scraped. + +``` + +``` diff --git a/benchmarks/main.ts b/benchmarks/main.ts index c00f9b9028..670be6f6e2 100644 --- a/benchmarks/main.ts +++ b/benchmarks/main.ts @@ -1,10 +1,16 @@ /// /// -import { Command } from "https://deno.land/x/cliffy@v0.25.2/command/mod.ts"; +import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts"; import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.38.5/mod.ts"; import { Action } from "./action.ts"; +import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts"; +import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts"; +export { + DenoLandProvider, + UpgradeCommand, +} from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts"; async function login(email: string, password: string): Promise { return await windmill.UserService.login({ @@ -15,10 +21,12 @@ async function login(email: string, password: string): Promise { }); } +export const VERSION = "v1.121.0"; + await new Command() - .name("windmill-bench") + .name("wmillbench") .description("Run Benchmark to measure throughput of windmill.") - .version("v0.0.0") + .version(VERSION) .option("--host ", "The windmill host to benchmark.", { default: "http://127.0.0.1:8000", }) @@ -83,6 +91,10 @@ await new Command() } ) .option("--use-flows", "Run flows instead of jobs.") + .option( + "--flow-pattern ", + "Use a different flow pattern among: 2steps, onebranch (Default 2steps)" + ) .option("--custom ", "Use custom actions during bench") .option( "--zombie-timeout", @@ -133,6 +145,7 @@ await new Command() histogramBuckets, maximumThroughput, useFlows, + flowPattern, zombieTimeout, continous, max, @@ -189,6 +202,7 @@ await new Command() exportSimple, maximumThroughput, useFlows, + flowPattern, zombieTimeout, }, null, @@ -226,6 +240,7 @@ await new Command() per_worker_throughput, max_per_worker, useFlows, + flowPattern, continous, custom: custom_content, }; @@ -385,4 +400,18 @@ await new Command() console.log("done"); } ) + .command( + "upgrade", + new UpgradeCommand({ + main: "main.ts", + args: [ + "--allow-net", + "--allow-read", + "--allow-write", + "--allow-env", + "--unstable", + ], + provider: new DenoLandProvider({ name: "wmillbench" }), + }) + ) .parse(); diff --git a/benchmarks/worker.ts b/benchmarks/worker.ts index 10e56b683b..74234e960d 100644 --- a/benchmarks/worker.ts +++ b/benchmarks/worker.ts @@ -10,6 +10,7 @@ const promise = new Promise<{ workspace_id: string; per_worker_throughput: number; useFlows: boolean; + flowPattern: string; continous: boolean; max_per_worker: number; custom: Action | undefined; @@ -23,6 +24,7 @@ const promise = new Promise<{ workspace_id: sharedConfig.workspace_id, per_worker_throughput: sharedConfig.per_worker_throughput, useFlows: sharedConfig.useFlows, + flowPattern: sharedConfig.flowPattern, continous: sharedConfig.continous, max_per_worker: sharedConfig.max_per_worker, custom: sharedConfig.custom, @@ -82,9 +84,46 @@ while (cont) { await evaluate(config.custom); continue; } else if (config.useFlows) { - uuid = await windmill.JobService.runFlowPreview({ - workspace: config.workspace_id, - requestBody: { + let payload: api.FlowPreview; + if (config.flowPattern == "branchone") { + payload = { + args: {}, + value: { + modules: [ + { + id: "a", + value: { + input_transforms: {}, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', + }, + }, + { + id: "b", + value: { + type: "branchone", + branches: [], + default: [ + { + id: "c", + value: { + input_transforms: {}, + language: api.RawScript.language.DENO, + type: "rawscript", + content: + 'export function main(){ return Deno.env.get("WM_JOB_ID"); }', + }, + }, + ], + }, + }, + ], + }, + }; + } else { + payload = { args: {}, value: { modules: [ @@ -110,7 +149,11 @@ while (cont) { }, ], }, - }, + }; + } + uuid = await windmill.JobService.runFlowPreview({ + workspace: config.workspace_id, + requestBody: payload, }); } else { uuid = await windmill.JobService.runScriptPreview({