From c28bfd4c63d5c9ccaae60b90d6a822816b8ec433 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 12 Jan 2023 23:03:21 +0000 Subject: [PATCH] Nightly Benchmarks: add user provided example (#3308) --- .github/workflows/benchmarking.yml | 105 ++++++++++++++++++++++ test_runner/performance/test_perf_olap.py | 39 ++++++++ 2 files changed, 144 insertions(+) diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 59317f0a47..16be60b1a1 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -489,3 +489,108 @@ jobs: slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + user-examples-compare: + if: success() || failure() + needs: [ tpch-compare ] + + strategy: + fail-fast: false + matrix: + # neon-captest-prefetch: We have pre-created projects with prefetch enabled + # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs + # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage + platform: [ neon-captest-prefetch, rds-postgres, rds-aurora ] + + env: + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install + DEFAULT_PG_VERSION: 14 + TEST_OUTPUT: /tmp/test_output + BUILD_TYPE: remote + SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }} + PLATFORM: ${{ matrix.platform }} + + runs-on: [ self-hosted, us-east-2, x64 ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init + + timeout-minutes: 360 # 6h + + steps: + - uses: actions/checkout@v3 + + - name: Download Neon artifact + uses: ./.github/actions/download + with: + name: neon-${{ runner.os }}-release-artifact + path: /tmp/neon/ + prefix: latest + + - name: Add Postgres binaries to PATH + run: | + ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version + echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH + + - name: Set up Connection String + id: set-up-connstr + run: | + case "${PLATFORM}" in + neon-captest-prefetch) + CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }} + ;; + rds-aurora) + CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_AURORA_CONNSTR }} + ;; + rds-postgres) + CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_POSTGRES_CONNSTR }} + ;; + *) + echo 2>&1 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-prefetch', 'rds-aurora', or 'rds-postgres'" + exit 1 + ;; + esac + + echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT + + psql ${CONNSTR} -c "SELECT version();" + + - name: Set database options + if: matrix.platform == 'neon-captest-prefetch' + run: | + DB_NAME=$(psql ${BENCHMARK_CONNSTR} --no-align --quiet -t -c "SELECT current_database()") + + psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET enable_seqscan_prefetch=on" + psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET effective_io_concurrency=32" + psql ${BENCHMARK_CONNSTR} -c "ALTER DATABASE ${DB_NAME} SET maintenance_io_concurrency=32" + env: + BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} + + - name: Run user examples + uses: ./.github/actions/run-python-test-set + with: + build_type: ${{ env.BUILD_TYPE }} + test_selection: performance/test_perf_olap.py + run_in_parallel: false + save_perf_report: ${{ env.SAVE_PERF_REPORT }} + extra_params: -m remote_cluster --timeout 21600 -k test_user_examples + env: + VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" + PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" + BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} + + - name: Create Allure report + if: success() || failure() + uses: ./.github/actions/allure-report + with: + action: generate + build_type: ${{ env.BUILD_TYPE }} + + - name: Post to a Slack channel + if: ${{ github.event.schedule && failure() }} + uses: slackapi/slack-github-action@v1 + with: + channel-id: "C033QLM5P7D" # dev-staging-stream + slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/test_runner/performance/test_perf_olap.py b/test_runner/performance/test_perf_olap.py index 8c3b6e57ff..39aafa80df 100644 --- a/test_runner/performance/test_perf_olap.py +++ b/test_runner/performance/test_perf_olap.py @@ -144,3 +144,42 @@ def test_tpch(query: LabelledQuery, remote_compare: RemoteCompare): """ run_psql(remote_compare, query, times=1) + + +@pytest.mark.remote_cluster +def test_user_examples(remote_compare: RemoteCompare): + query = LabelledQuery( + "Q1", + r""" + SELECT + v20.c2263 AS v1, + v19.c2484 AS v2, + DATE_TRUNC('month', v18.c37)::DATE AS v3, + (ARRAY_AGG(c1840 order by v18.c37))[1] AS v4, + (ARRAY_AGG(c1841 order by v18.c37 DESC))[1] AS v5, + SUM(v17.c1843) AS v6, + SUM(v17.c1844) AS v7, + SUM(v17.c1848) AS v8, + SUM(v17.c1845) AS v9, + SUM(v17.c1846) AS v10, + SUM(v17.c1861) AS v11, + SUM(v17.c1860) AS v12, + SUM(v17.c1869) AS v13, + SUM(v17.c1856) AS v14, + SUM(v17.c1855) AS v15, + SUM(v17.c1854) AS v16 + FROM + s3.t266 v17 + INNER JOIN s1.t41 v18 ON v18.c34 = v17.c1836 + INNER JOIN s3.t571 v19 ON v19.c2482 = v17.c1834 + INNER JOIN s3.t331 v20 ON v20.c2261 = v17.c1835 + WHERE + (v17.c1835 = 4) AND + (v18.c37 >= '2019-03-01') AND + (v17.c1833 = 2) + GROUP BY v1, v2, v3 + ORDER BY v1, v2, v3 + LIMIT 199; + """, + ) + run_psql(remote_compare, query, times=3)