From fde16f86140deeefd300cf8bf3fc17dd93cfa22d Mon Sep 17 00:00:00 2001 From: Fedor Dikarev Date: Mon, 11 Nov 2024 21:33:29 +0100 Subject: [PATCH] use batch gh-workflow-stats-action with separate table (#9722) We found that exporting GH Workflow Runs in batch is more efficient due to - better utilisation of Github API - and gh runners usage is rounded to minutes, so even when ad-hoc export is done in 5-10 seconds, we billed for one minute usage So now we introduce batch exporting, with version v0.2.x of github workflow stats exporter. How it's expected to work now: - every 15 minutes we query for the workflow runs, created in last 2 hours - to avoid missing workflows that ran for more than 2 hours, every night (00:25) we will query workflows created in past 24 hours and export them as well - should we have query for even longer periods? - lets see how it works with current schedule - for longer periods like for days or weeks, it may require to adjust logic and concurrency of querying data, so lets for now use simpler version --- .../workflows/report-workflow-stats-batch.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 .github/workflows/report-workflow-stats-batch.yml diff --git a/.github/workflows/report-workflow-stats-batch.yml b/.github/workflows/report-workflow-stats-batch.yml new file mode 100644 index 0000000000..98e394a3c2 --- /dev/null +++ b/.github/workflows/report-workflow-stats-batch.yml @@ -0,0 +1,29 @@ +name: Report Workflow Stats Batch + +on: + schedule: + - cron: '*/15 * * * *' + - cron: '25 0 * * *' + +jobs: + gh-workflow-stats-batch: + name: GitHub Workflow Stats Batch + runs-on: ubuntu-22.04 + permissions: + actions: read + steps: + - name: Export Workflow Run for the past 2 hours + uses: neondatabase/gh-workflow-stats-action@v0.2.1 + with: + db_uri: ${{ secrets.GH_REPORT_STATS_DB_RW_CONNSTR }} + db_table: "gh_workflow_stats_batch_neon" + gh_token: ${{ secrets.GITHUB_TOKEN }} + duration: '2h' + - name: Export Workflow Run for the past 24 hours + if: github.event.schedule == '25 0 * * *' + uses: neondatabase/gh-workflow-stats-action@v0.2.1 + with: + db_uri: ${{ secrets.GH_REPORT_STATS_DB_RW_CONNSTR }} + db_table: "gh_workflow_stats_batch_neon" + gh_token: ${{ secrets.GITHUB_TOKEN }} + duration: '24h'