diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 4c0be0d480..efaaba1506 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -34,24 +34,18 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - uses: actions/checkout@v4 - with: - ref: benchmarks - name: benchmark timeout-minutes: 20 run: deno run --unstable -A -r https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json - - name: Push changes - run: | - pwd - git pull origin benchmarks - git add . - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git commit -m "Update single worker benchmarks" - git push + - name: Save benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_single + path: | + *.json benchmark_dedicated: runs-on: ubicloud-standard-8 @@ -81,24 +75,18 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - uses: actions/checkout@v4 - with: - ref: benchmarks - name: benchmark timeout-minutes: 20 run: deno run --unstable -A -r https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts --no-warm-up -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json - - name: Push changes - run: | - pwd - git pull origin benchmarks - git add . - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git commit -m "Update dedicated worker benchmarks" - git push + - name: Save benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_dedicated + path: | + *.json benchmark_4workers: runs-on: ubicloud-standard-8 @@ -161,9 +149,6 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - uses: actions/checkout@v4 - with: - ref: benchmarks - name: benchmark timeout-minutes: 20 run: deno run --unstable -A -r @@ -171,15 +156,12 @@ jobs: -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json --workers 4 - - name: Push changes - run: | - pwd - git pull origin benchmarks - git add . - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git commit -m "Update 4 workers benchmarks" - git push + - name: Save benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_4workers + path: | + *.json benchmark_8workers: runs-on: ubicloud-standard-8 @@ -285,9 +267,6 @@ jobs: - uses: denoland/setup-deno@v1 with: deno-version: v1.x - - uses: actions/checkout@v4 - with: - ref: benchmarks - name: benchmark timeout-minutes: 20 run: deno run --unstable -A -r @@ -295,15 +274,12 @@ jobs: -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json --workers 8 - - name: Push changes - run: | - pwd - git pull origin benchmarks - git add . - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git commit -m "Update 8 workers benchmarks" - git push + - name: Save benchmark results + uses: actions/upload-artifact@v4 + with: + name: benchmark_8workers + path: | + *.json benchmark_graphs: runs-on: ubicloud @@ -319,6 +295,10 @@ jobs: - uses: actions/checkout@v4 with: ref: benchmarks + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + merge-multiple: true - name: graphs run: deno run --unstable -A -r https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts @@ -326,10 +306,10 @@ jobs: https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json - name: Push changes run: | + ls -la pwd - git pull origin benchmarks git add . git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --local user.name "github-actions[bot]" - git commit -m "Update graphs" + git commit -m "Update benchmarks" git push diff --git a/benchmarks/benchmark_suite.ts b/benchmarks/benchmark_suite.ts index e4ea1aa92a..e03148ce99 100644 --- a/benchmarks/benchmark_suite.ts +++ b/benchmarks/benchmark_suite.ts @@ -9,6 +9,7 @@ import { VERSION } from "./lib.ts"; type Config = { kind: string; jobs: number; + noSave?: boolean; }[]; async function warmUp( @@ -79,6 +80,10 @@ async function main({ jobs: benchmark.jobs, }); + if (benchmark.noSave) { + continue; + } + if (!result) { throw new Error("No result returned"); } @@ -91,8 +96,10 @@ async function main({ benchmark.kind + (workers > 1 ? `_${workers}workers` : ""); const jsonFilePath = `${benchmarkName}_benchmark.json`; try { - const existing = await Deno.readTextFile(jsonFilePath); - data = JSON.parse(existing); + const remotePath = + "https://raw.githubusercontent.com/windmill-labs/windmill/benchmarks/" + + jsonFilePath; + data = await fetch(remotePath).then((r) => r.json()); } catch (_) { console.log("No existing data file found, creating new one."); } diff --git a/benchmarks/suite_config.json b/benchmarks/suite_config.json index f58f13c1cb..568402ca2d 100644 --- a/benchmarks/suite_config.json +++ b/benchmarks/suite_config.json @@ -1,41 +1,33 @@ [ { - "graph_title": "noop throughput benchmark", "kind": "noop", "jobs": 90000 }, { - "graph_title": "flow throughput benchmark", "kind": "2steps", "jobs": 250 }, { - "graph_title": "deno throughput benchmark", "kind": "deno", "jobs": 500 }, { - "graph_title": "bun throughput benchmark", "kind": "bun", "jobs": 500 }, { - "graph_title": "python throughput benchmark", "kind": "python", "jobs": 500 }, { - "graph_title": "go throughput benchmark", "kind": "go", "jobs": 2000 }, { - "graph_title": "bash throughput benchmark", "kind": "bash", "jobs": 2000 }, { - "graph_title": "nativets throughput benchmark", "kind": "nativets", "jobs": 2000 } diff --git a/benchmarks/suite_dedicated.json b/benchmarks/suite_dedicated.json index 0d83b444b3..56fe022424 100644 --- a/benchmarks/suite_dedicated.json +++ b/benchmarks/suite_dedicated.json @@ -1,11 +1,10 @@ [ { - "graph_title": "warmup", "kind": "dedicated", - "jobs": 50000 + "jobs": 50000, + "noSave": true }, { - "graph_title": "dedicated throughput benchmark", "kind": "dedicated", "jobs": 90000 }