Store Allure tests statistics in database (#2367)

.github/actions/allure-report/action.yml

@@ -11,6 +11,10 @@ inputs:
   test_selection:
     description: '`test_selector` from run-python-test-set action'
     required: false
+outputs:
+  report-url:
+    description: 'Allure report URL'
+    value: ${{ steps.generate-report.outputs.report-url }}
 
 runs:
   using: "composite"
@@ -182,7 +186,7 @@ runs:
         aws s3 cp --only-show-errors ./index.html "s3://${BUCKET}/${REPORT_PREFIX}/latest/index.html"
 
         echo "[Allure Report](${REPORT_URL})" >> ${GITHUB_STEP_SUMMARY}
-        echo "::set-output name=REPORT_URL::${REPORT_URL}"
+        echo "::set-output name=report-url::${REPORT_URL}"
 
     - name: Release Allure lock
       if: ${{ inputs.action == 'generate' && always() }}
@@ -200,7 +204,7 @@ runs:
     - uses: actions/github-script@v6
       if: ${{ inputs.action == 'generate' && always() }}
       env:
-        REPORT_URL: ${{ steps.generate-report.outputs.REPORT_URL }}
+        REPORT_URL: ${{ steps.generate-report.outputs.report-url }}
         BUILD_TYPE: ${{ inputs.build_type }}
         SHA: ${{ github.event.pull_request.head.sha || github.sha }}
       with:

.github/workflows/build_and_test.yml

@@ -253,6 +253,7 @@ jobs:
           real_s3_region: us-west-2
           real_s3_access_key_id: "${{ secrets.AWS_ACCESS_KEY_ID_CI_TESTS_S3 }}"
           real_s3_secret_access_key: "${{ secrets.AWS_SECRET_ACCESS_KEY_CI_TESTS_S3 }}"
+
       - name: Merge and upload coverage data
         if: matrix.build_type == 'debug'
         uses: ./.github/actions/save-coverage-data
@@ -308,11 +309,28 @@ jobs:
           submodules: false
 
       - name: Create Allure report
+        id: create-allure-report
         uses: ./.github/actions/allure-report
         with:
           action: generate
           build_type: ${{ matrix.build_type }}
 
+      - name: Store Allure test stat in the DB
+        env:
+          BUILD_TYPE: ${{ matrix.build_type }}
+          SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          REPORT_URL: ${{ steps.create-allure-report.outputs.report-url }}
+          TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR }}
+        shell: bash -euxo pipefail {0}
+        run: |
+          curl --fail --output suites.json ${REPORT_URL%/index.html}/data/suites.json
+          ./scripts/pysync
+
+          # Workaround for https://github.com/neondatabase/cloud/issues/2188
+          psql "$TEST_RESULT_CONNSTR" -c "SELECT 1;" || sleep 10
+
+          DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json
+
   coverage-report:
     runs-on: dev
     container:

scripts/ingest_perf_test_result.py

@@ -2,6 +2,7 @@
 import argparse
 import json
 import os
+import sys
 from contextlib import contextmanager
 from datetime import datetime
 from pathlib import Path
@@ -26,7 +27,7 @@ CREATE TABLE IF NOT EXISTS perf_test_results (
 
 def err(msg):
     print(f"error: {msg}")
-    exit(1)
+    sys.exit(1)
 
 
 @contextmanager
@@ -43,8 +44,8 @@ def create_table(cur):
     cur.execute(CREATE_TABLE)
 
 
-def ingest_perf_test_result(cursor, data_dile: Path, recorded_at_timestamp: int) -> int:
-    run_data = json.loads(data_dile.read_text())
+def ingest_perf_test_result(cursor, data_file: Path, recorded_at_timestamp: int) -> int:
+    run_data = json.loads(data_file.read_text())
     revision = run_data["revision"]
     platform = run_data["platform"]
 

scripts/ingest_regress_test_result.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+
+import psycopg2
+
+CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS regress_test_results (
+    id SERIAL PRIMARY KEY,
+    reference CHAR(255),
+    revision CHAR(40),
+    build_type CHAR(16),
+    data JSONB
+)
+"""
+
+
+def err(msg):
+    print(f"error: {msg}")
+    sys.exit(1)
+
+
+@contextmanager
+def get_connection_cursor():
+    connstr = os.getenv("DATABASE_URL")
+    if not connstr:
+        err("DATABASE_URL environment variable is not set")
+    with psycopg2.connect(connstr, connect_timeout=30) as conn:
+        with conn.cursor() as cur:
+            yield cur
+
+
+def create_table(cur):
+    cur.execute(CREATE_TABLE)
+
+
+def ingest_regress_test_result(
+    cursor, reference: str, revision: str, build_type: str, data_file: Path
+):
+    values = (
+        reference,
+        revision,
+        build_type,
+        data_file.read_text(),
+    )
+    cursor.execute(
+        """
+        INSERT INTO regress_test_results (
+            reference,
+            revision,
+            build_type,
+            data
+        ) VALUES (%s, %s, %s, %s)
+        """,
+        values,
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Regress test result uploader. \
+            Database connection string should be provided via DATABASE_URL environment variable",
+    )
+    parser.add_argument("--initdb", action="store_true", help="Initialuze database")
+    parser.add_argument(
+        "--reference", type=str, required=True, help="git reference, for example refs/heads/main"
+    )
+    parser.add_argument("--revision", type=str, required=True, help="git revision")
+    parser.add_argument(
+        "--build-type", type=str, required=True, help="build type: release, debug or remote"
+    )
+    parser.add_argument(
+        "--ingest", type=Path, required=True, help="Path to regress test result file"
+    )
+
+    args = parser.parse_args()
+    with get_connection_cursor() as cur:
+        if args.initdb:
+            create_table(cur)
+
+        if not args.ingest.exists():
+            err(f"ingest path {args.ingest} does not exist")
+
+        ingest_regress_test_result(
+            cur,
+            reference=args.reference,
+            revision=args.revision,
+            build_type=args.build_type,
+            data_file=args.ingest,
+        )
+
+
+if __name__ == "__main__":
+    main()