nit improve benchmarks

.github/workflows/benchmark.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  benchmark:
+  benchmark_single:
     runs-on: ubicloud-standard-8
     services:
       postgres:
@@ -30,16 +30,6 @@ jobs:
           http://localhost:8000/api/version"
         ports:
           - 8000:8000
-      windmill-worker:
-        image: ghcr.io/windmill-labs/windmill-ee:main
-        env:
-          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
-          MODE: worker
-          WORKER_GROUP: dedicated
-          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
-          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
-        options: >-
-          --pull always --restart unless-stopped
     steps:
       - uses: denoland/setup-deno@v1
         with:
@@ -62,3 +52,120 @@ jobs:
           git config --local user.name "github-actions[bot]"
           git commit -m "Update benchmarks"
           git push
+
+  benchmark_dedicated:
+    runs-on: ubicloud-standard-8
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+      windmill:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: dedicated
+          DEDICATED_WORKER: "admins:f/benchmarks/dedicated"
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+        ports:
+          - 8000:8000
+    steps:
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+      - uses: actions/checkout@v4
+        with:
+          ref: benchmarks
+      - name: benchmark
+        timeout-minutes: 20
+        run:
+          deno run --unstable -A -r
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
+          -c
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json
+
+  benchmark_4workers:
+    runs-on: ubicloud-standard-8
+    services:
+      postgres:
+        image: postgres
+        env:
+          POSTGRES_DB: windmill
+          POSTGRES_PASSWORD: changeme
+        options: >-
+          --health-cmd pg_isready --health-interval 10s --health-timeout 5s
+          --health-retries 5
+      windmill:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+        ports:
+          - 8000:8000
+      windmill_1:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          MODE: worker
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+
+      windmill_2:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          MODE: worker
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+
+      windmill_3:
+        image: ghcr.io/windmill-labs/windmill-ee:main
+        env:
+          DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill
+          LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }}
+          MODE: worker
+          WORKER_GROUP: main
+          WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets
+        options: >-
+          --pull always --health-interval 10s --health-timeout 5s
+          --health-retries 5 --health-cmd "curl
+          http://localhost:8000/api/version"
+
+    steps:
+      - uses: denoland/setup-deno@v1
+        with:
+          deno-version: v1.x
+      - uses: actions/checkout@v4
+        with:
+          ref: benchmarks
+      - name: benchmark
+        timeout-minutes: 20
+        run:
+          deno run --unstable -A -r
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts
+          -c
+          https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json

benchmarks/suite_config.json

@@ -10,11 +10,6 @@
       "kind": "2steps",
       "jobs": 250
     },
-    {
-      "graph_title": "dedicated throughput benchmark (single worker)",
-      "kind": "dedicated",
-      "jobs": 90000
-    },
     {
       "graph_title": "deno throughput benchmark (single worker)",
       "kind": "deno",
@@ -52,8 +47,8 @@
       "kinds": ["go", "python", "deno", "bun", "bash"]
     },
     {
-      "graph_title": "bun vs dedicated vs noop",
-      "kinds": ["bun", "dedicated", "noop"]
+      "graph_title": "bun vs noop",
+      "kinds": ["bun", "noop"]
     }
   ]
 }