diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b48ad1c990..b4ed577946 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -19,12 +19,13 @@ jobs: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 --shm-size=2g + --network host windmill: image: ghcr.io/windmill-labs/windmill-ee:main env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill + DATABASE_URL: postgres://postgres:changeme@localhost:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets @@ -44,281 +45,3 @@ jobs: https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts -c https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json - - name: Save benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark_single - path: | - *.json - - benchmark_dedicated: - runs-on: ubicloud-standard-8 - services: - postgres: - image: postgres - env: - POSTGRES_DB: windmill - POSTGRES_PASSWORD: changeme - POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" - options: >- - --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 - windmill: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - WORKER_GROUP: dedicated - DEDICATED_WORKER: "admins:f/benchmarks/dedicated" - options: >- - --pull always --restart unless-stopped --health-interval 10s --health-timeout 5s - --health-retries 5 --health-cmd "curl - http://localhost:8000/api/version" - ports: - - 8000:8000 - steps: - - uses: denoland/setup-deno@v2 - with: - deno-version: v1.x - - name: benchmark - timeout-minutes: 20 - run: deno run --unstable -A -r - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts - --no-warm-up -c - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_dedicated.json - - name: Save benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark_dedicated - path: | - *.json - - benchmark_4workers: - runs-on: ubicloud-standard-8 - services: - postgres: - image: postgres - env: - POSTGRES_DB: windmill - POSTGRES_PASSWORD: changeme - POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" - options: >- - --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 - windmill: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always --health-interval 10s --health-timeout 5s - --health-retries 5 --health-cmd "curl - http://localhost:8000/api/version" - ports: - - 8000:8000 - windmill_1: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_2: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_3: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - steps: - - uses: denoland/setup-deno@v2 - with: - deno-version: v1.x - - name: benchmark - timeout-minutes: 20 - run: deno run --unstable -A -r - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts - -c - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json - --workers 4 - --factor 3 - - name: Save benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark_4workers - path: | - *.json - - benchmark_8workers: - runs-on: ubicloud-standard-8 - services: - postgres: - image: postgres - env: - POSTGRES_DB: windmill - POSTGRES_PASSWORD: changeme - POSTGRES_INITDB_ARGS: "-c shared_buffers=2GB -c work_mem=32MB -c effective_cache_size=4GB" - options: >- - --health-cmd pg_isready --health-interval 10s --health-timeout 5s - --health-retries 5 - windmill: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always --health-interval 10s --health-timeout 5s - --health-retries 5 --health-cmd "curl - http://localhost:8000/api/version" - ports: - - 8000:8000 - windmill_1: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_2: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_3: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_4: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_5: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_6: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - - windmill_7: - image: ghcr.io/windmill-labs/windmill-ee:main - env: - DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill - LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} - MODE: worker - WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets - options: >- - --pull always - steps: - - uses: denoland/setup-deno@v2 - with: - deno-version: v1.x - - name: benchmark - timeout-minutes: 20 - run: deno run --unstable -A -r - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_suite.ts - -c - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/suite_config.json - --workers 8 - --factor 3 - - name: Save benchmark results - uses: actions/upload-artifact@v4 - with: - name: benchmark_8workers - path: | - *.json - - benchmark_graphs: - runs-on: ubicloud - needs: - - benchmark_single - - benchmark_dedicated - - benchmark_4workers - - benchmark_8workers - steps: - - uses: denoland/setup-deno@v2 - with: - deno-version: v1.x - - uses: actions/checkout@v4 - with: - ref: benchmarks - - name: Download benchmark results - uses: actions/download-artifact@v4 - with: - merge-multiple: true - - name: graphs - run: deno run --unstable -A -r - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/benchmark_graphs.ts - -c - https://raw.githubusercontent.com/windmill-labs/windmill/${GITHUB_REF##ref/head/}/benchmarks/graphs_config.json - - name: Push changes - run: | - ls -la - pwd - git add . - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git commit -m "Update benchmarks" - git push