starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-08-21 08:02:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: release wmillbench publicly

Browse Source
This commit is contained in:
Ruben Fiszel
2023-06-23 15:08:33 +02:00
parent c341345103
commit 161f793ae6
4 changed files with 136 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/change-versions.sh
+1
View File
@@ -5,6 +5,7 @@ echo "Updating versions to: $VERSION"
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" cli/main.ts
sed -i -e "/^export const VERSION =/s/= .*/= \"v$VERSION\";/" benchmarks/main.ts
sed -i -e "/version: /s/: .*/: $VERSION/" backend/windmill-api/openapi.yaml
sed -i -e "/version: /s/: .*/: $VERSION/" openflow.openapi.yaml
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
benchmarks/README.md
+56 -31
View File
@@ -1,47 +1,72 @@
# Benchmarks
This folder includes a small deno/ts utility to benchmark execution of jobs & flows.
This folder includes a small deno/ts utility to benchmark execution of jobs &
flows.
## Installation
Install the `wmill` CLI tool using
`deno install --unstable -A https://deno.land/x/wmillbench/main.ts`.
Update to the latest version using `wmillbench upgrade`.
## Quickstart
Have your instance expose prometheus metrics (METRICS_ADDR=1).
Then
```
Usage: windmill-bench
wmillbench -s 1 -e admin@windmill.dev -p changeme --host YOUR_HOST
```
## Usage
```
Usage: wmillbench
Description:
Run Benchmark to measure throughput of windmill.
Run Benchmark to measure throughput of windmill.
Options:
-h, --help - Show this help.
-V, --version - Show the version number for this program.
--host <url> - The windmill host to benchmark. (Default: "http://127.0.0.1/")
--workers <workers> - The number of workers to run at once. (Default: 1)
-s, --seconds <seconds> - How long to run the benchmark for (in seconds). (Default: 30)
-e, --email <email> - The email to use to login.
-p, --password <password> - The password to use to login.
-t, --token <token> - The token to use when talking to the API server. Preferred over manual login.
-w, --workspace <workspace> - The workspace to spawn scripts from. (Default: "starter")
-m, --metrics <metrics> - The url to scrape metrics from. (Default: "http://localhost:8001/metrics")
--export-json <export_json> - If set, exports will be into a JSON file.
--export-csv <export_csv> - If set, exports will be into a csv file.
--export-histograms [histograms...] - Mark metrics (without label) that are reported as histograms to export.
--export-simple [simple...] - Mark metrics (without label) that are reported as simple values.
--maximum-throughput <maximum_throughput> - Maximum number of jobs/flows to start in one second. (Default: Infinity)
--use-flows - Run flows instead of jobs.
--histogram-buckets [buckets...] - Define what buckets to collect from histograms. (Default: [
"+Inf", "10", "5",
"2.5", "2.5", "1",
"0.5", "0.25", "0.1",
"0.05", "0.025", "0.01",
"0.005"
])
-h, --help - Show this help. -V, --version - Show the version number for this
program. --host <url> - The windmill host to benchmark. (Default:
"http://127.0.0.1:8000/") --workers <workers> - The number of workers to run at
once. (Default: 1) -s, --seconds <seconds> - How long to run the benchmark for
(in seconds). (Default: 30) -e, --email <email> - The email to use to login. -p,
--password <password> - The password to use to login. -t, --token <token> - The
token to use when talking to the API server. Preferred over manual login. -w,
--workspace <workspace> - The workspace to spawn scripts from. (Default:
"starter") -m, --metrics <metrics> - The url to scrape metrics from. (Default:
"http://localhost:8001/metrics") --export-json <export_json> - If set, exports
will be into a JSON file. --export-csv <export_csv> - If set, exports will be
into a csv file. --export-histograms [histograms...] - Mark metrics (without
label) that are reported as histograms to export. --export-simple [simple...] -
Mark metrics (without label) that are reported as simple values.
--maximum-throughput <maximum_throughput> - Maximum number of jobs/flows to
start in one second. (Default: Infinity) --use-flows - Run flows instead of
jobs. --histogram-buckets [buckets...] - Define what buckets to collect from
histograms. (Default: [ "+Inf", "10", "5", "2.5", "2.5", "1", "0.5", "0.25",
"0.1", "0.05", "0.025", "0.01", "0.005" ])
Environment variables:
WM_TOKEN <token> - The token to use when talking to the API server. Preferred over manual login.
WM_WORKSPACE <workspace> - The workspace to spawn scripts from.
WM_TOKEN <token> - The token to use when talking to the API server. Preferred
over manual login. WM_WORKSPACE <workspace> - The workspace to spawn scripts
from.
```
This will run a simple benchmark against localhost (the default admin email + password are set above), all execution is done in the "bench" workspace (as set via `--workspace`).
This will run a simple benchmark against localhost (the default admin email +
password are set above), all execution is done in the "bench" workspace (as set
via `--workspace`).
Metrics are exported to JSON will only include mean & stdev, histograms get one entry for each bucket.
CSV will include a full list of all values scraped.
Metrics are exported to JSON will only include mean & stdev, histograms get one
entry for each bucket. CSV will include a full list of all values scraped.
```
```
benchmarks/main.ts
+32 -3
View File
@@ -1,10 +1,16 @@
/// <reference no-default-lib="true" />
/// <reference lib="deno.window" />
import { Command } from "https://deno.land/x/cliffy@v0.25.2/command/mod.ts";
import { Command } from "https://deno.land/x/cliffy@v0.25.7/command/mod.ts";
import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.38.5/mod.ts";
import { Action } from "./action.ts";
import { UpgradeCommand } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/upgrade_command.ts";
import { DenoLandProvider } from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
export {
DenoLandProvider,
UpgradeCommand,
} from "https://deno.land/x/cliffy@v0.25.7/command/upgrade/mod.ts";
async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
@@ -15,10 +21,12 @@ async function login(email: string, password: string): Promise<string> {
});
}
export const VERSION = "v1.121.0";
await new Command()
.name("windmill-bench")
.name("wmillbench")
.description("Run Benchmark to measure throughput of windmill.")
.version("v0.0.0")
.version(VERSION)
.option("--host <url:string>", "The windmill host to benchmark.", {
default: "http://127.0.0.1:8000",
})
@@ -83,6 +91,10 @@ await new Command()
}
)
.option("--use-flows", "Run flows instead of jobs.")
.option(
"--flow-pattern <pattern:string>",
"Use a different flow pattern among: 2steps, onebranch (Default 2steps)"
)
.option("--custom <custom_path:string>", "Use custom actions during bench")
.option(
"--zombie-timeout",
@@ -133,6 +145,7 @@ await new Command()
histogramBuckets,
maximumThroughput,
useFlows,
flowPattern,
zombieTimeout,
continous,
max,
@@ -189,6 +202,7 @@ await new Command()
exportSimple,
maximumThroughput,
useFlows,
flowPattern,
zombieTimeout,
},
null,
@@ -226,6 +240,7 @@ await new Command()
per_worker_throughput,
max_per_worker,
useFlows,
flowPattern,
continous,
custom: custom_content,
};
@@ -385,4 +400,18 @@ await new Command()
console.log("done");
}
)
.command(
"upgrade",
new UpgradeCommand({
main: "main.ts",
args: [
"--allow-net",
"--allow-read",
"--allow-write",
"--allow-env",
"--unstable",
],
provider: new DenoLandProvider({ name: "wmillbench" }),
})
)
.parse();
benchmarks/worker.ts
+47 -4
View File
@@ -10,6 +10,7 @@ const promise = new Promise<{
workspace_id: string;
per_worker_throughput: number;
useFlows: boolean;
flowPattern: string;
continous: boolean;
max_per_worker: number;
custom: Action | undefined;
@@ -23,6 +24,7 @@ const promise = new Promise<{
workspace_id: sharedConfig.workspace_id,
per_worker_throughput: sharedConfig.per_worker_throughput,
useFlows: sharedConfig.useFlows,
flowPattern: sharedConfig.flowPattern,
continous: sharedConfig.continous,
max_per_worker: sharedConfig.max_per_worker,
custom: sharedConfig.custom,
@@ -82,9 +84,46 @@ while (cont) {
await evaluate(config.custom);
continue;
} else if (config.useFlows) {
uuid = await windmill.JobService.runFlowPreview({
workspace: config.workspace_id,
requestBody: {
let payload: api.FlowPreview;
if (config.flowPattern == "branchone") {
payload = {
args: {},
value: {
modules: [
{
id: "a",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
},
},
{
id: "b",
value: {
type: "branchone",
branches: [],
default: [
{
id: "c",
value: {
input_transforms: {},
language: api.RawScript.language.DENO,
type: "rawscript",
content:
'export function main(){ return Deno.env.get("WM_JOB_ID"); }',
},
},
],
},
},
],
},
};
} else {
payload = {
args: {},
value: {
modules: [
@@ -110,7 +149,11 @@ while (cont) {
},
],
},
},
};
}
uuid = await windmill.JobService.runFlowPreview({
workspace: config.workspace_id,
requestBody: payload,
});
} else {
uuid = await windmill.JobService.runScriptPreview({