From a25fa29bc922e2b29cdfd97f76c93b9a93ac406a Mon Sep 17 00:00:00 2001 From: Dmitry Rodionov Date: Fri, 28 Jan 2022 12:41:05 +0300 Subject: [PATCH] modify git-upload for generate_and_push_perf_report.sh needs --- .circleci/config.yml | 3 +-- .github/workflows/benchmarking.yml | 2 +- scripts/generate_and_push_perf_report.sh | 33 ++++++++++-------------- scripts/git-upload | 33 +++++++++++++++++++++--- 4 files changed, 44 insertions(+), 27 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1f0bf1765c..97b91c50a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -297,7 +297,7 @@ jobs: - PLATFORM: zenith-local-ci command: | PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)" - rm -rf $PERF_REPORT_DIR || true + rm -rf $PERF_REPORT_DIR TEST_SELECTION="test_runner/<< parameters.test_selection >>" EXTRA_PARAMS="<< parameters.extra_params >>" @@ -342,7 +342,6 @@ jobs: if << parameters.save_perf_report >>; then if [[ $CIRCLE_BRANCH == "main" ]]; then - # TODO: reuse scripts/git-upload export REPORT_FROM="$PERF_REPORT_DIR" export REPORT_TO=local scripts/generate_and_push_perf_report.sh diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index b7e1149cb8..dfa11839d4 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -91,7 +91,7 @@ jobs: run: | # just to be sure that no data was cached on self hosted runner # since it might generate duplicates when calling ingest_perf_test_result.py - rm -rf perf-report-staging || true + rm -rf perf-report-staging mkdir -p perf-report-staging ./scripts/pytest test_runner/performance/ -v -m "remote_cluster" --skip-interfering-proc-check --out-dir perf-report-staging diff --git a/scripts/generate_and_push_perf_report.sh b/scripts/generate_and_push_perf_report.sh index 3ddf0f9beb..798bb1ff34 100755 --- a/scripts/generate_and_push_perf_report.sh +++ b/scripts/generate_and_push_perf_report.sh @@ -1,30 +1,23 @@ #!/bin/bash # this is a shortcut script to avoid duplication in CI - set -eux -o pipefail SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -git clone --single-branch --branch testing https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git # TODO (PRIOR MERGE) replace back before merge -cd zenith-perf-data -mkdir -p reports/ -mkdir -p data/$REPORT_TO - +echo "Uploading perf report to zenith pg" # ingest per test results data into zenith backed postgres running in staging to build grafana reports on that data -DATABASE_URL=$PERF_TEST_RESULT_CONNSTR poetry run python $SCRIPT_DIR/ingest_perf_test_result.py --ingest $REPORT_FROM +DATABASE_URL="$PERF_TEST_RESULT_CONNSTR" poetry run python "$SCRIPT_DIR"/ingest_perf_test_result.py --ingest "$REPORT_FROM" -cp $REPORT_FROM/* data/$REPORT_TO +# Activate poetry's venv. Needed because git upload does not run in a project dir (it uses tmp to store the repository) +# so the problem occurs because poetry cannot find pyproject.toml in temp dir created by git upload +# shellcheck source=/dev/null +. "$(poetry env info --path)"/bin/activate -echo "Generating report" -poetry run python $SCRIPT_DIR/generate_perf_report_page.py --input-dir data/$REPORT_TO --out reports/$REPORT_TO.html -echo "Uploading perf result" -git add data reports -git \ - -c "user.name=vipvap" \ - -c "user.email=vipvap@zenith.tech" \ - commit \ - --author="vipvap " \ - -m "add performance test result for $GITHUB_SHA zenith revision" - -git push https://$VIP_VAP_ACCESS_TOKEN@github.com/zenithdb/zenith-perf-data.git testing # master TODO (PRIOR MERGE) replace back before merge +echo "Uploading perf result to zenith-perf-data" +scripts/git-upload \ + --repo=https://"$VIP_VAP_ACCESS_TOKEN"@github.com/zenithdb/zenith-perf-data.git \ + --message="add performance test result for $GITHUB_SHA zenith revision" \ + --branch=testing `# TODO (PRIOR MERGE) replace back with master`\ + copy "$REPORT_FROM" "data/$REPORT_TO" `# COPY FROM TO_RELATIVE`\ + --run-cmd "python $SCRIPT_DIR/generate_perf_report_page.py --input-dir data/$REPORT_TO --out reports/$REPORT_TO.html" diff --git a/scripts/git-upload b/scripts/git-upload index db554b48f7..5449575df7 100755 --- a/scripts/git-upload +++ b/scripts/git-upload @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from contextlib import contextmanager +import shlex from tempfile import TemporaryDirectory from pathlib import Path @@ -9,6 +10,8 @@ import os import shutil import subprocess import sys +import textwrap +from typing import Optional def absolute_path(path): @@ -38,13 +41,21 @@ def run(cmd, *args, **kwargs): class GitRepo: - def __init__(self, url): + def __init__(self, url, branch: Optional[str] = None): self.url = url self.cwd = TemporaryDirectory() + self.branch = branch - subprocess.check_call([ + args = [ 'git', 'clone', + '--single-branch', + ] + if self.branch: + args.extend(['--branch', self.branch]) + + subprocess.check_call([ + *args, str(url), self.cwd.name, ]) @@ -105,18 +116,32 @@ def do_copy(args): else: shutil.copy(src, dst) + if args.run_cmd: + run(shlex.split(args.run_cmd)) + def main(): parser = argparse.ArgumentParser(description='Git upload tool') parser.add_argument('--repo', type=str, metavar='URL', required=True, help='git repo url') parser.add_argument('--message', type=str, metavar='TEXT', help='commit message') + parser.add_argument('--branch', type=str, metavar='TEXT', help='target git repo branch') commands = parser.add_subparsers(title='commands', dest='subparser_name') - p_copy = commands.add_parser('copy', help='copy file into the repo') + p_copy = commands.add_parser( + 'copy', + help='copy file into the repo', + formatter_class=argparse.RawTextHelpFormatter, + ) p_copy.add_argument('src', type=absolute_path, help='source path') p_copy.add_argument('dst', type=relative_path, help='relative dest path') p_copy.add_argument('--forbid-overwrite', action='store_true', help='do not allow overwrites') + p_copy.add_argument('--run-cmd', + help=textwrap.dedent('''\ + run arbitrary cmd on top of copied files, + example usage is static content generation + based on current repository state\ + ''')) args = parser.parse_args() @@ -127,7 +152,7 @@ def main(): action = commands.get(args.subparser_name) if action: message = args.message or 'update' - GitRepo(args.repo).update(message, lambda: action(args)) + GitRepo(args.repo, args.branch).update(message, lambda: action(args)) else: parser.print_usage()