diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index e2f15d96db..a0701bdbcf 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -1,9 +1,9 @@ name: Benchmarking on: - # uncomment to run on push for debugging your PR - # push: - # branches: [ your branch ] + push: + branches: + - bayandin/pg16-bench schedule: # * is a special character in YAML so you have to quote this string # ┌───────────── minute (0 - 59) @@ -39,10 +39,10 @@ jobs: TEST_PG_BENCH_DURATIONS_MATRIX: "300" TEST_PG_BENCH_SCALES_MATRIX: "10,100" POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 + DEFAULT_PG_VERSION: 16 TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote - SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} + SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'bayandin/pg16-bench' ) }} PLATFORM: "neon-staging" runs-on: [ self-hosted, us-east-2, x64 ] @@ -116,8 +116,6 @@ jobs: runs-on: ubuntu-latest outputs: pgbench-compare-matrix: ${{ steps.pgbench-compare-matrix.outputs.matrix }} - olap-compare-matrix: ${{ steps.olap-compare-matrix.outputs.matrix }} - tpch-compare-matrix: ${{ steps.tpch-compare-matrix.outputs.matrix }} steps: - name: Generate matrix for pgbench benchmark @@ -143,41 +141,6 @@ jobs: echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT - - name: Generate matrix for OLAP benchmarks - id: olap-compare-matrix - run: | - matrix='{ - "platform": [ - "neon-captest-reuse" - ] - }' - - if [ "$(date +%A)" = "Saturday" ]; then - matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres" }, - { "platform": "rds-aurora" }]') - fi - - echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT - - - name: Generate matrix for TPC-H benchmarks - id: tpch-compare-matrix - run: | - matrix='{ - "platform": [ - "neon-captest-reuse" - ], - "scale": [ - "10" - ] - }' - - if [ "$(date +%A)" = "Saturday" ]; then - matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "scale": "10" }, - { "platform": "rds-aurora", "scale": "10" }]') - fi - - echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT - pgbench-compare: needs: [ generate-matrices ] @@ -189,10 +152,10 @@ jobs: TEST_PG_BENCH_DURATIONS_MATRIX: "60m" TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }} POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 + DEFAULT_PG_VERSION: 16 TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote - SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} + SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'bayandin/pg16-bench' ) }} PLATFORM: ${{ matrix.platform }} runs-on: [ self-hosted, us-east-2, x64 ] @@ -215,8 +178,11 @@ jobs: - name: Add Postgres binaries to PATH run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH + # We don't have Postgres 16 binaries in the Neon artifact yet, so use pgbench from v15 + # (the binaries will be uploaded after the release) + + ${POSTGRES_DISTRIB_DIR}/v15/bin/pgbench --version + echo "${POSTGRES_DISTRIB_DIR}/v15/bin" >> $GITHUB_PATH - name: Create Neon Project if: contains(fromJson('["neon-captest-new", "neon-captest-freetier", "neonvm-captest-new", "neonvm-captest-freetier"]'), matrix.platform) @@ -317,293 +283,3 @@ jobs: slack-message: "Periodic perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - - clickbench-compare: - # ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters - # we use for performance testing in pgbench-compare. - # Run this job only when pgbench-compare is finished to avoid the intersection. - # We might change it after https://github.com/neondatabase/neon/issues/2900. - # - # *_CLICKBENCH_CONNSTR: Genuine ClickBench DB with ~100M rows - # *_CLICKBENCH_10M_CONNSTR: DB with the first 10M rows of ClickBench DB - if: ${{ !cancelled() }} - needs: [ generate-matrices, pgbench-compare ] - - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }} - - env: - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 - TEST_OUTPUT: /tmp/test_output - BUILD_TYPE: remote - SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} - PLATFORM: ${{ matrix.platform }} - - runs-on: [ self-hosted, us-east-2, x64 ] - container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned - options: --init - - steps: - - uses: actions/checkout@v3 - - - name: Download Neon artifact - uses: ./.github/actions/download - with: - name: neon-${{ runner.os }}-release-artifact - path: /tmp/neon/ - prefix: latest - - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - - name: Set up Connection String - id: set-up-connstr - run: | - case "${PLATFORM}" in - neon-captest-reuse) - CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR }} - ;; - rds-aurora) - CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CLICKBENCH_10M_CONNSTR }} - ;; - rds-postgres) - CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CLICKBENCH_10M_CONNSTR }} - ;; - *) - echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'rds-aurora', or 'rds-postgres'" - exit 1 - ;; - esac - - echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - - QUERY="SELECT version();" - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;" - fi - psql ${CONNSTR} -c "${QUERY}" - - - name: ClickBench benchmark - uses: ./.github/actions/run-python-test-set - with: - build_type: ${{ env.BUILD_TYPE }} - test_selection: performance/test_perf_olap.py - run_in_parallel: false - save_perf_report: ${{ env.SAVE_PERF_REPORT }} - extra_params: -m remote_cluster --timeout 21600 -k test_clickbench - env: - VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" - PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" - BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} - TEST_OLAP_SCALE: 10 - - - name: Create Allure report - if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report-generate - - - name: Post to a Slack channel - if: ${{ github.event.schedule && failure() }} - uses: slackapi/slack-github-action@v1 - with: - channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Periodic OLAP perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - - tpch-compare: - # TCP-H DB for rds-aurora and rds-Postgres deployed to the same clusters - # we use for performance testing in pgbench-compare & clickbench-compare. - # Run this job only when clickbench-compare is finished to avoid the intersection. - # We might change it after https://github.com/neondatabase/neon/issues/2900. - # - # *_TPCH_S10_CONNSTR: DB generated with scale factor 10 (~10 GB) - if: ${{ !cancelled() }} - needs: [ generate-matrices, clickbench-compare ] - - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.generate-matrices.outputs.tpch-compare-matrix) }} - - env: - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 - TEST_OUTPUT: /tmp/test_output - BUILD_TYPE: remote - SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} - PLATFORM: ${{ matrix.platform }} - TEST_OLAP_SCALE: ${{ matrix.scale }} - - runs-on: [ self-hosted, us-east-2, x64 ] - container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned - options: --init - - steps: - - uses: actions/checkout@v3 - - - name: Download Neon artifact - uses: ./.github/actions/download - with: - name: neon-${{ runner.os }}-release-artifact - path: /tmp/neon/ - prefix: latest - - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - - name: Get Connstring Secret Name - run: | - case "${PLATFORM}" in - neon-captest-reuse) - ENV_PLATFORM=CAPTEST_TPCH - ;; - rds-aurora) - ENV_PLATFORM=RDS_AURORA_TPCH - ;; - rds-postgres) - ENV_PLATFORM=RDS_AURORA_TPCH - ;; - *) - echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'rds-aurora', or 'rds-postgres'" - exit 1 - ;; - esac - - CONNSTR_SECRET_NAME="BENCHMARK_${ENV_PLATFORM}_S${TEST_OLAP_SCALE}_CONNSTR" - echo "CONNSTR_SECRET_NAME=${CONNSTR_SECRET_NAME}" >> $GITHUB_ENV - - - name: Set up Connection String - id: set-up-connstr - run: | - CONNSTR=${{ secrets[env.CONNSTR_SECRET_NAME] }} - - echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - - QUERY="SELECT version();" - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;" - fi - psql ${CONNSTR} -c "${QUERY}" - - - name: Run TPC-H benchmark - uses: ./.github/actions/run-python-test-set - with: - build_type: ${{ env.BUILD_TYPE }} - test_selection: performance/test_perf_olap.py - run_in_parallel: false - save_perf_report: ${{ env.SAVE_PERF_REPORT }} - extra_params: -m remote_cluster --timeout 21600 -k test_tpch - env: - VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" - PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" - BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} - TEST_OLAP_SCALE: ${{ matrix.scale }} - - - name: Create Allure report - if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report-generate - - - name: Post to a Slack channel - if: ${{ github.event.schedule && failure() }} - uses: slackapi/slack-github-action@v1 - with: - channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} - - user-examples-compare: - if: ${{ !cancelled() }} - needs: [ generate-matrices, tpch-compare ] - - strategy: - fail-fast: false - matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }} - - env: - POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 14 - TEST_OUTPUT: /tmp/test_output - BUILD_TYPE: remote - SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }} - PLATFORM: ${{ matrix.platform }} - - runs-on: [ self-hosted, us-east-2, x64 ] - container: - image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned - options: --init - - steps: - - uses: actions/checkout@v3 - - - name: Download Neon artifact - uses: ./.github/actions/download - with: - name: neon-${{ runner.os }}-release-artifact - path: /tmp/neon/ - prefix: latest - - - name: Add Postgres binaries to PATH - run: | - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version - echo "${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin" >> $GITHUB_PATH - - - name: Set up Connection String - id: set-up-connstr - run: | - case "${PLATFORM}" in - neon-captest-reuse) - CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }} - ;; - rds-aurora) - CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_AURORA_CONNSTR }} - ;; - rds-postgres) - CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_POSTGRES_CONNSTR }} - ;; - *) - echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neon-captest-reuse', 'rds-aurora', or 'rds-postgres'" - exit 1 - ;; - esac - - echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT - - QUERY="SELECT version();" - if [[ "${PLATFORM}" = "neon"* ]]; then - QUERY="${QUERY} SHOW neon.tenant_id; SHOW neon.timeline_id;" - fi - psql ${CONNSTR} -c "${QUERY}" - - - name: Run user examples - uses: ./.github/actions/run-python-test-set - with: - build_type: ${{ env.BUILD_TYPE }} - test_selection: performance/test_perf_olap.py - run_in_parallel: false - save_perf_report: ${{ env.SAVE_PERF_REPORT }} - extra_params: -m remote_cluster --timeout 21600 -k test_user_examples - env: - VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}" - PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}" - BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }} - - - name: Create Allure report - if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report-generate - - - name: Post to a Slack channel - if: ${{ github.event.schedule && failure() }} - uses: slackapi/slack-github-action@v1 - with: - channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Periodic User example perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}