Save performance test results from perfirmance test suit runs.

Also render reports for both staging and local runs.

.circleci/config.yml

@@ -225,6 +225,9 @@ jobs:
       run_in_parallel:
         type: boolean
         default: true
+      save_perf_report:
+        type: boolean
+        default: false
     steps:
       - attach_workspace:
           at: /tmp/zenith
@@ -248,6 +251,9 @@ jobs:
             - ZENITH_BIN: /tmp/zenith/bin
             - POSTGRES_DISTRIB_DIR: /tmp/zenith/pg_install
             - TEST_OUTPUT: /tmp/test_output
+            # this variable will be embedded in perf test report 
+            # and is needed to distinguish different environments
+            - PLATFORM: zenith-local-ci
           command: |
             TEST_SELECTION="<< parameters.test_selection >>"
             EXTRA_PARAMS="<< parameters.extra_params >>"
@@ -258,6 +264,15 @@ jobs:
             if << parameters.run_in_parallel >>; then
               EXTRA_PARAMS="-n4 $EXTRA_PARAMS"
             fi;
+            if << parameters.save_perf_report >>; then
+              if [[ $CIRCLE_BRANCH == "main" ]]; then
+                mkdir -p perf-report-local
+                EXTRA_PARAMS="--out-dir perf-report-local $EXTRA_PARAMS"
+              fi;
+            fi;
+
+            export GITHUB_SHA=$CIRCLE_SHA1
+
             # Run the tests.
             #
             # The junit.xml file allows CircleCI to display more fine-grained test information
@@ -269,6 +284,13 @@ jobs:
             # -s is not used to prevent pytest from capturing output, because tests are running
             # in parallel and logs are mixed between different tests
             pipenv run pytest --junitxml=$TEST_OUTPUT/junit.xml --tb=short --verbose -m "not remote_cluster" -rA $TEST_SELECTION $EXTRA_PARAMS
+
+            if << parameters.save_perf_report >>; then
+              if [[ $CIRCLE_BRANCH == "main" ]]; then
+                REPORT_FROM=$(realpath perf-report-local) REPORT_TO=local ../scripts/generate_and_push_perf_report.sh
+              fi;
+            fi;
+
       - run:
           # CircleCI artifacts are preserved one file at a time, so skipping
           # this step isn't a good idea. If you want to extract the
@@ -385,6 +407,7 @@ workflows:
           build_type: release
           test_selection: performance
           run_in_parallel: false
+          save_perf_report: true
           requires:
             - build-zenith-release
       - docker-image:

.github/workflows/benchmarking.yml

@@ -35,14 +35,6 @@ jobs:
     - name: Checkout zenith repo
       uses: actions/checkout@v2
 
-    - name: Checkout zenith-perf-data repo
-      uses: actions/checkout@v2
-      with:
-        repository: zenithdb/zenith-perf-data
-        token: ${{ secrets.VIP_VAP_ACCESS_TOKEN }}
-        ref: master
-        path: zenith-perf-data
-
     # actions/setup-python@v2 is not working correctly on self-hosted runners
     # see https://github.com/actions/setup-python/issues/162
     # and probably https://github.com/actions/setup-python/issues/162#issuecomment-865387976 in particular
@@ -112,14 +104,11 @@ jobs:
         BENCHMARK_CONNSTR: "${{ secrets.BENCHMARK_STAGING_CONNSTR }}"
         REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally
       run: |
-        mkdir -p zenith-perf-data/data/staging
-        pipenv run pytest test_runner/performance/ -v -m "remote_cluster" --skip-interfering-proc-check --out-dir zenith-perf-data/data/staging
+        mkdir -p perf-report-staging
+        pipenv run pytest test_runner/performance/ -v -m "remote_cluster" --skip-interfering-proc-check --out-dir perf-report-staging
 
     - name: Submit result
       env:
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
       run: |
-        cd zenith-perf-data
-        git add data
-        git commit --author="vipvap <vipvap@zenith.tech>" -m "add performance test result for $GITHUB_SHA zenith revision"
-        git push https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git master
+        REPORT_FROM=$(realpath perf-report-staging) REPORT_TO=staging scripts/generate_and_push_perf_report.sh