modify git-upload for generate_and_push_perf_report.sh needs

.circleci/config.yml

@@ -297,7 +297,7 @@ jobs:
             - PLATFORM: zenith-local-ci
           command: |
             PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)"
-            rm -rf $PERF_REPORT_DIR || true
+            rm -rf $PERF_REPORT_DIR
 
             TEST_SELECTION="test_runner/<< parameters.test_selection >>"
             EXTRA_PARAMS="<< parameters.extra_params >>"
@@ -342,7 +342,6 @@ jobs:
 
             if << parameters.save_perf_report >>; then
               if [[ $CIRCLE_BRANCH == "main" ]]; then
-                # TODO: reuse scripts/git-upload
                 export REPORT_FROM="$PERF_REPORT_DIR"
                 export REPORT_TO=local
                 scripts/generate_and_push_perf_report.sh

.github/workflows/benchmarking.yml

@@ -91,7 +91,7 @@ jobs:
       run: |
         # just to be sure that no data was cached on self hosted runner
         # since it might generate duplicates when calling ingest_perf_test_result.py
-        rm -rf perf-report-staging || true
+        rm -rf perf-report-staging
         mkdir -p perf-report-staging
         ./scripts/pytest test_runner/performance/ -v -m "remote_cluster" --skip-interfering-proc-check --out-dir perf-report-staging
 

scripts/generate_and_push_perf_report.sh

@@ -1,30 +1,23 @@
 #!/bin/bash
 
 # this is a shortcut script to avoid duplication in CI
-
 set -eux -o pipefail
 
 SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
 
-git clone --single-branch --branch testing https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git  # TODO (PRIOR MERGE) replace back before merge
-cd zenith-perf-data
-mkdir -p reports/
-mkdir -p data/$REPORT_TO
-
+echo "Uploading perf report to zenith pg"
 # ingest per test results data into zenith backed postgres running in staging to build grafana reports on that data
-DATABASE_URL=$PERF_TEST_RESULT_CONNSTR poetry run python $SCRIPT_DIR/ingest_perf_test_result.py --ingest $REPORT_FROM
+DATABASE_URL="$PERF_TEST_RESULT_CONNSTR" poetry run python "$SCRIPT_DIR"/ingest_perf_test_result.py --ingest "$REPORT_FROM"
 
-cp $REPORT_FROM/* data/$REPORT_TO
+# Activate poetry's venv. Needed because git upload does not run in a project dir (it uses tmp to store the repository)
+# so the problem occurs because poetry cannot find pyproject.toml in temp dir created by git upload
+# shellcheck source=/dev/null
+. "$(poetry env info --path)"/bin/activate
 
-echo "Generating report"
-poetry run python $SCRIPT_DIR/generate_perf_report_page.py --input-dir data/$REPORT_TO --out reports/$REPORT_TO.html
-echo "Uploading perf result"
-git add data reports
-git \
-    -c "user.name=vipvap" \
-    -c "user.email=vipvap@zenith.tech" \
-    commit \
-    --author="vipvap <vipvap@zenith.tech>" \
-    -m "add performance test result for $GITHUB_SHA zenith revision"
-
-git push https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git testing # master TODO (PRIOR MERGE) replace back before merge
+echo "Uploading perf result to zenith-perf-data"
+scripts/git-upload \
+    --repo=https://"$VIP_VAP_ACCESS_TOKEN"@github.com/zenithdb/zenith-perf-data.git \
+    --message="add performance test result for $GITHUB_SHA zenith revision" \
+    --branch=testing `# TODO (PRIOR MERGE) replace back with master`\
+    copy "$REPORT_FROM" "data/$REPORT_TO" `# COPY FROM TO_RELATIVE`\
+    --run-cmd "python $SCRIPT_DIR/generate_perf_report_page.py --input-dir data/$REPORT_TO --out reports/$REPORT_TO.html"

scripts/git-upload

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 from contextlib import contextmanager
+import shlex
 from tempfile import TemporaryDirectory
 from pathlib import Path
 
@@ -9,6 +10,8 @@ import os
 import shutil
 import subprocess
 import sys
+import textwrap
+from typing import Optional
 
 
 def absolute_path(path):
@@ -38,13 +41,21 @@ def run(cmd, *args, **kwargs):
 
 
 class GitRepo:
-    def __init__(self, url):
+    def __init__(self, url, branch: Optional[str] = None):
         self.url = url
         self.cwd = TemporaryDirectory()
+        self.branch = branch
 
-        subprocess.check_call([
+        args = [
             'git',
             'clone',
+            '--single-branch',
+        ]
+        if self.branch:
+            args.extend(['--branch', self.branch])
+
+        subprocess.check_call([
+            *args,
             str(url),
             self.cwd.name,
         ])
@@ -105,18 +116,32 @@ def do_copy(args):
     else:
         shutil.copy(src, dst)
 
+    if args.run_cmd:
+        run(shlex.split(args.run_cmd))
+
 
 def main():
     parser = argparse.ArgumentParser(description='Git upload tool')
     parser.add_argument('--repo', type=str, metavar='URL', required=True, help='git repo url')
     parser.add_argument('--message', type=str, metavar='TEXT', help='commit message')
+    parser.add_argument('--branch', type=str, metavar='TEXT', help='target git repo branch')
 
     commands = parser.add_subparsers(title='commands', dest='subparser_name')
 
-    p_copy = commands.add_parser('copy', help='copy file into the repo')
+    p_copy = commands.add_parser(
+        'copy',
+        help='copy file into the repo',
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
     p_copy.add_argument('src', type=absolute_path, help='source path')
     p_copy.add_argument('dst', type=relative_path, help='relative dest path')
     p_copy.add_argument('--forbid-overwrite', action='store_true', help='do not allow overwrites')
+    p_copy.add_argument('--run-cmd',
+                        help=textwrap.dedent('''\
+                run arbitrary cmd on top of copied files,
+                example usage is static content generation
+                based on current repository state\
+            '''))
 
     args = parser.parse_args()
 
@@ -127,7 +152,7 @@ def main():
     action = commands.get(args.subparser_name)
     if action:
         message = args.message or 'update'
-        GitRepo(args.repo).update(message, lambda: action(args))
+        GitRepo(args.repo, args.branch).update(message, lambda: action(args))
     else:
         parser.print_usage()