mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
6689d6fd89
https://github.com/neondatabase/cloud/issues/19011 - Prewarm config changes are not publicly available. Correct the test by using a pre-filled 50 GB project on staging - Create extension neon with schema neon to fix read performance tests on staging, error example in https://neon-github-public-dev.s3.amazonaws.com/reports/main/16483462789/index.html#suites/3d632da6dda4a70f5b4bd24904ab444c/919841e331089fc4/ - Don't create extra endpoint in LFC prewarm performance tests
1292 lines
53 KiB
YAML
1292 lines
53 KiB
YAML
name: Benchmarking
|
|
|
|
on:
|
|
# uncomment to run on push for debugging your PR
|
|
# push:
|
|
# branches: [ your branch ]
|
|
schedule:
|
|
# * is a special character in YAML so you have to quote this string
|
|
# ┌───────────── minute (0 - 59)
|
|
# │ ┌───────────── hour (0 - 23)
|
|
# │ │ ┌───────────── day of the month (1 - 31)
|
|
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
|
|
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
|
|
- cron: '0 3 * * *' # run once a day, timezone is utc
|
|
workflow_dispatch: # adds ability to run this manually
|
|
inputs:
|
|
region_id:
|
|
description: 'Project region id. If not set, the default region will be used'
|
|
required: false
|
|
default: 'aws-us-east-2'
|
|
save_perf_report:
|
|
type: boolean
|
|
description: 'Publish perf report. If not set, the report will be published only for the main branch'
|
|
required: false
|
|
collect_olap_explain:
|
|
type: boolean
|
|
description: 'Collect EXPLAIN ANALYZE for OLAP queries. If not set, EXPLAIN ANALYZE will not be collected'
|
|
required: false
|
|
default: false
|
|
collect_pg_stat_statements:
|
|
type: boolean
|
|
description: 'Collect pg_stat_statements for OLAP queries. If not set, pg_stat_statements will not be collected'
|
|
required: false
|
|
default: false
|
|
run_AWS_RDS_AND_AURORA:
|
|
type: boolean
|
|
description: 'AWS-RDS and AWS-AURORA normally only run on Saturday. Set this to true to run them on every workflow_dispatch'
|
|
required: false
|
|
default: false
|
|
run_only_pgvector_tests:
|
|
type: boolean
|
|
description: 'Run pgvector tests but no other tests. If not set, all tests including pgvector tests will be run'
|
|
required: false
|
|
default: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash -euxo pipefail {0}
|
|
|
|
concurrency:
|
|
# Allow only one workflow per any non-`main` branch.
|
|
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
cleanup:
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
env:
|
|
ORG_ID: org-solitary-dew-09443886
|
|
LIMIT: 100
|
|
SEARCH: "GITHUB_RUN_ID="
|
|
BASE_URL: https://console-stage.neon.build/api/v2
|
|
DRY_RUN: "false" # Set to "true" to just test out the workflow
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Cleanup inactive Neon projects left over from prior runs
|
|
env:
|
|
API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
run: |
|
|
set -euo pipefail
|
|
|
|
NOW=$(date -u +%s)
|
|
DAYS_AGO=$((NOW - 5 * 86400))
|
|
|
|
REQUEST_URL="$BASE_URL/projects?limit=$LIMIT&search=$(printf '%s' "$SEARCH" | jq -sRr @uri)&org_id=$ORG_ID"
|
|
|
|
echo "Requesting project list from:"
|
|
echo "$REQUEST_URL"
|
|
|
|
response=$(curl -s -X GET "$REQUEST_URL" \
|
|
--header "Accept: application/json" \
|
|
--header "Content-Type: application/json" \
|
|
--header "Authorization: Bearer ${API_KEY}" )
|
|
|
|
echo "Response:"
|
|
echo "$response" | jq .
|
|
|
|
projects_to_delete=$(echo "$response" | jq --argjson cutoff "$DAYS_AGO" '
|
|
.projects[]
|
|
| select(.compute_last_active_at != null)
|
|
| select((.compute_last_active_at | fromdateiso8601) < $cutoff)
|
|
| {id, name, compute_last_active_at}
|
|
')
|
|
|
|
if [ -z "$projects_to_delete" ]; then
|
|
echo "No projects eligible for deletion."
|
|
exit 0
|
|
fi
|
|
|
|
echo "Projects that will be deleted:"
|
|
echo "$projects_to_delete" | jq -r '.id'
|
|
|
|
if [ "$DRY_RUN" = "false" ]; then
|
|
echo "$projects_to_delete" | jq -r '.id' | while read -r project_id; do
|
|
echo "Deleting project: $project_id"
|
|
curl -s -X DELETE "$BASE_URL/projects/$project_id" \
|
|
--header "Accept: application/json" \
|
|
--header "Content-Type: application/json" \
|
|
--header "Authorization: Bearer ${API_KEY}"
|
|
done
|
|
else
|
|
echo "Dry run enabled — no projects were deleted."
|
|
fi
|
|
bench:
|
|
if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- PG_VERSION: 16
|
|
PLATFORM: "neon-staging"
|
|
region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
|
|
RUNNER: [ self-hosted, us-east-2, x64 ]
|
|
- PG_VERSION: 17
|
|
PLATFORM: "neon-staging"
|
|
region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
|
|
RUNNER: [ self-hosted, us-east-2, x64 ]
|
|
- PG_VERSION: 16
|
|
PLATFORM: "azure-staging"
|
|
region_id: 'azure-eastus2'
|
|
RUNNER: [ self-hosted, eastus2, x64 ]
|
|
env:
|
|
TEST_PG_BENCH_DURATIONS_MATRIX: "300"
|
|
TEST_PG_BENCH_SCALES_MATRIX: "10,100"
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
PG_VERSION: ${{ matrix.PG_VERSION }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.PLATFORM }}
|
|
|
|
runs-on: ${{ matrix.RUNNER }}
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials # necessary on Azure runners
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Create Neon Project
|
|
id: create-neon-project
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
region_id: ${{ matrix.region_id }}
|
|
postgres_version: ${{ env.PG_VERSION }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Run benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
# Set --sparse-ordering option of pytest-order plugin
|
|
# to ensure tests are running in order of appears in the file.
|
|
# It's important for test_perf_pgbench.py::test_pgbench_remote_* tests
|
|
extra_params:
|
|
-m remote_cluster
|
|
--sparse-ordering
|
|
--timeout 14400
|
|
--ignore test_runner/performance/test_perf_olap.py
|
|
--ignore test_runner/performance/test_perf_pgvector_queries.py
|
|
--ignore test_runner/performance/test_logical_replication.py
|
|
--ignore test_runner/performance/test_physical_replication.py
|
|
--ignore test_runner/performance/test_perf_ingest_using_pgcopydb.py
|
|
--ignore test_runner/performance/test_cumulative_statistics_persistence.py
|
|
--ignore test_runner/performance/test_perf_many_relations.py
|
|
--ignore test_runner/performance/test_perf_oltp_large_tenant.py
|
|
--ignore test_runner/performance/test_lfc_prewarm.py
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Delete Neon Project
|
|
if: ${{ always() }}
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Periodic perf testing: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
cumstats-test:
|
|
if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 17
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: "neon-staging"
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Verify that cumulative statistics are preserved
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_cumulative_statistics_persistence.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 3600
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
replication-tests:
|
|
if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 16
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: "neon-staging"
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Run Logical Replication benchmarks
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_logical_replication.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 5400
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
BENCHMARK_PROJECT_ID_PUB: ${{ vars.BENCHMARK_PROJECT_ID_PUB }}
|
|
BENCHMARK_PROJECT_ID_SUB: ${{ vars.BENCHMARK_PROJECT_ID_SUB }}
|
|
|
|
- name: Run Physical Replication benchmarks
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_physical_replication.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 5400
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
store-test-results-into-db: true
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
|
|
|
|
# Post both success and failure to the Slack channel
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && !cancelled() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06T9AMNDQQ" # on-call-compute-staging-stream
|
|
slack-message: |
|
|
Periodic replication testing: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
prewarm-test:
|
|
if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
env:
|
|
PROJECT_ID: ${{ vars.PREWARM_PROJECT_ID }}
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 17
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: "neon-staging"
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Run prewarm benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_lfc_prewarm.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 5400
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
store-test-results-into-db: true
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
|
|
|
|
generate-matrices:
|
|
if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
|
|
# Create matrices for the benchmarking jobs, so we run benchmarks on rds only once a week (on Saturday)
|
|
#
|
|
# Available platforms:
|
|
# - neonvm-captest-new: Freshly created project (1 CU)
|
|
# - neonvm-captest-freetier: Use freetier-sized compute (0.25 CU)
|
|
# - neonvm-captest-azure-new: Freshly created project (1 CU) in azure region
|
|
# - neonvm-captest-azure-freetier: Use freetier-sized compute (0.25 CU) in azure region
|
|
# - neonvm-captest-reuse: Reusing existing project
|
|
# - rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
|
|
# - rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
|
|
env:
|
|
RUN_AWS_RDS_AND_AURORA: ${{ github.event.inputs.run_AWS_RDS_AND_AURORA || 'false' }}
|
|
DEFAULT_REGION_ID: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
|
|
runs-on: ubuntu-22.04
|
|
outputs:
|
|
pgbench-compare-matrix: ${{ steps.pgbench-compare-matrix.outputs.matrix }}
|
|
olap-compare-matrix: ${{ steps.olap-compare-matrix.outputs.matrix }}
|
|
tpch-compare-matrix: ${{ steps.tpch-compare-matrix.outputs.matrix }}
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Generate matrix for pgbench benchmark
|
|
id: pgbench-compare-matrix
|
|
run: |
|
|
region_id_default=${{ env.DEFAULT_REGION_ID }}
|
|
runner_default='["self-hosted", "us-east-2", "x64"]'
|
|
runner_azure='["self-hosted", "eastus2", "x64"]'
|
|
image_default="ghcr.io/neondatabase/build-tools:pinned-bookworm"
|
|
matrix='{
|
|
"pg_version" : [
|
|
16
|
|
],
|
|
"region_id" : [
|
|
"'"$region_id_default"'"
|
|
],
|
|
"platform": [
|
|
"neonvm-captest-new",
|
|
"neonvm-captest-reuse",
|
|
"neonvm-captest-new"
|
|
],
|
|
"db_size": [ "10gb" ],
|
|
"runner": ['"$runner_default"'],
|
|
"image": [ "'"$image_default"'" ],
|
|
"include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"', "image": "ghcr.io/neondatabase/build-tools:pinned-bookworm" },
|
|
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb","runner": '"$runner_azure"', "image": "ghcr.io/neondatabase/build-tools:pinned-bookworm" },
|
|
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb","runner": '"$runner_azure"', "image": "ghcr.io/neondatabase/build-tools:pinned-bookworm" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new-many-tables","db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 17, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]
|
|
}'
|
|
|
|
if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
|
|
matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "rds-postgres", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "rds-aurora", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]')
|
|
fi
|
|
|
|
echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Generate matrix for OLAP benchmarks
|
|
id: olap-compare-matrix
|
|
run: |
|
|
matrix='{
|
|
"platform": [
|
|
"neonvm-captest-reuse"
|
|
],
|
|
"pg_version" : [
|
|
16,17
|
|
]
|
|
}'
|
|
|
|
if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
|
|
matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 16, "platform": "rds-postgres" },
|
|
{ "pg_version": 16, "platform": "rds-aurora" }]')
|
|
fi
|
|
|
|
echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Generate matrix for TPC-H benchmarks
|
|
id: tpch-compare-matrix
|
|
run: |
|
|
matrix='{
|
|
"platform": [
|
|
"neonvm-captest-reuse"
|
|
],
|
|
"pg_version" : [
|
|
16,17
|
|
]
|
|
}'
|
|
|
|
if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
|
|
matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 16, "platform": "rds-postgres" },
|
|
{ "pg_version": 16, "platform": "rds-aurora" }]')
|
|
fi
|
|
|
|
echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
|
|
|
|
prepare_AWS_RDS_databases:
|
|
uses: ./.github/workflows/_benchmarking_preparation.yml
|
|
secrets: inherit
|
|
|
|
pgbench-compare:
|
|
if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
|
|
needs: [ generate-matrices, prepare_AWS_RDS_databases ]
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{fromJSON(needs.generate-matrices.outputs.pgbench-compare-matrix)}}
|
|
|
|
env:
|
|
TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
|
|
TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }}
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
PG_VERSION: ${{ matrix.pg_version }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: ${{ matrix.runner }}
|
|
container:
|
|
image: ${{ matrix.image }}
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
# Increase timeout to 8h, default timeout is 6h
|
|
timeout-minutes: 480
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Create Neon Project
|
|
if: contains(fromJSON('["neonvm-captest-new", "neonvm-captest-new-many-tables", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform)
|
|
id: create-neon-project
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
region_id: ${{ matrix.region_id }}
|
|
postgres_version: ${{ env.PG_VERSION }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
compute_units: ${{ (contains(matrix.platform, 'captest-freetier') && '[0.25, 0.25]') || '[1, 1]' }}
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR }}
|
|
;;
|
|
neonvm-captest-sharding-reuse)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_SHARDING_CONNSTR }}
|
|
;;
|
|
neonvm-captest-new | neonvm-captest-new-many-tables | neonvm-captest-freetier | neonvm-azure-captest-new | neonvm-azure-captest-freetier)
|
|
CONNSTR=${{ steps.create-neon-project.outputs.dsn }}
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CONNSTR }}
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
# we want to compare Neon project OLTP throughput and latency at scale factor 10 GB
|
|
# without (neonvm-captest-new)
|
|
# and with (neonvm-captest-new-many-tables) many relations in the database
|
|
- name: Create many relations before the run
|
|
if: contains(fromJSON('["neonvm-captest-new-many-tables"]'), matrix.platform)
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_perf_many_relations
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
TEST_NUM_RELATIONS: 10000
|
|
|
|
- name: Benchmark init
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Benchmark simple-update
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Benchmark select-only
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Delete Neon Project
|
|
if: ${{ steps.create-neon-project.outputs.project_id && always() }}
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Periodic perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
pgbench-pgvector:
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- PLATFORM: "neonvm-captest-pgvector"
|
|
RUNNER: [ self-hosted, us-east-2, x64 ]
|
|
postgres_version: 16
|
|
- PLATFORM: "neonvm-captest-pgvector-pg17"
|
|
RUNNER: [ self-hosted, us-east-2, x64 ]
|
|
postgres_version: 17
|
|
- PLATFORM: "azure-captest-pgvector"
|
|
RUNNER: [ self-hosted, eastus2, x64 ]
|
|
postgres_version: 16
|
|
|
|
env:
|
|
TEST_PG_BENCH_DURATIONS_MATRIX: "15m"
|
|
TEST_PG_BENCH_SCALES_MATRIX: "1"
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
PG_VERSION: ${{ matrix.postgres_version }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.PLATFORM }}
|
|
|
|
runs-on: ${{ matrix.RUNNER }}
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-pgvector)
|
|
CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }}
|
|
;;
|
|
neonvm-captest-pgvector-pg17)
|
|
CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_PG17 }}
|
|
;;
|
|
azure-captest-pgvector)
|
|
CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_AZURE }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Benchmark pgvector hnsw indexing
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
|
|
- name: Benchmark pgvector queries
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_pgvector_queries.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Periodic perf testing on ${{ env.PLATFORM }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
clickbench-compare:
|
|
# ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters
|
|
# we use for performance testing in pgbench-compare.
|
|
# Run this job only when pgbench-compare is finished to avoid the intersection.
|
|
# We might change it after https://github.com/neondatabase/neon/issues/2900.
|
|
#
|
|
# *_CLICKBENCH_CONNSTR: Genuine ClickBench DB with ~100M rows
|
|
# *_CLICKBENCH_10M_CONNSTR: DB with the first 10M rows of ClickBench DB
|
|
if: ${{ !cancelled() && (github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null) }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
needs: [ generate-matrices, pgbench-compare, prepare_AWS_RDS_databases ]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJSON(needs.generate-matrices.outputs.olap-compare-matrix) }}
|
|
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
PG_VERSION: ${{ matrix.pg_version }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
TEST_OLAP_COLLECT_EXPLAIN: ${{ github.event.inputs.collect_olap_explain }}
|
|
TEST_OLAP_COLLECT_PG_STAT_STATEMENTS: ${{ github.event.inputs.collect_pg_stat_statements }}
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
# Increase timeout to 12h, default timeout is 6h
|
|
# we have regression in clickbench causing it to run 2-3x longer
|
|
timeout-minutes: 720
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
case "${PG_VERSION}" in
|
|
16)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR }}
|
|
;;
|
|
17)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_CONNSTR_PG17 }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CLICKBENCH_10M_CONNSTR }}
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CLICKBENCH_10M_CONNSTR }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: ClickBench benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 43200 -k test_clickbench
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
TEST_OLAP_COLLECT_EXPLAIN: ${{ github.event.inputs.collect_olap_explain || 'false' }}
|
|
TEST_OLAP_COLLECT_PG_STAT_STATEMENTS: ${{ github.event.inputs.collect_pg_stat_statements || 'false' }}
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
TEST_OLAP_SCALE: 10
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Periodic OLAP perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
tpch-compare:
|
|
# TCP-H DB for rds-aurora and rds-Postgres deployed to the same clusters
|
|
# we use for performance testing in pgbench-compare & clickbench-compare.
|
|
# Run this job only when clickbench-compare is finished to avoid the intersection.
|
|
# We might change it after https://github.com/neondatabase/neon/issues/2900.
|
|
#
|
|
# *_TPCH_S10_CONNSTR: DB generated with scale factor 10 (~10 GB)
|
|
# if: ${{ !cancelled() && (github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null) }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
needs: [ generate-matrices, clickbench-compare, prepare_AWS_RDS_databases ]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJSON(needs.generate-matrices.outputs.tpch-compare-matrix) }}
|
|
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
PG_VERSION: ${{ matrix.pg_version }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Get Connstring Secret Name
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
case "${PG_VERSION}" in
|
|
16)
|
|
CONNSTR_SECRET_NAME="BENCHMARK_CAPTEST_TPCH_S10_CONNSTR"
|
|
;;
|
|
17)
|
|
CONNSTR_SECRET_NAME="BENCHMARK_CAPTEST_TPCH_CONNSTR_PG17"
|
|
;;
|
|
*)
|
|
echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR_SECRET_NAME="BENCHMARK_RDS_AURORA_TPCH_S10_CONNSTR"
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR_SECRET_NAME="BENCHMARK_RDS_POSTGRES_TPCH_S10_CONNSTR"
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "CONNSTR_SECRET_NAME=${CONNSTR_SECRET_NAME}" >> $GITHUB_ENV
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
CONNSTR=${{ secrets[env.CONNSTR_SECRET_NAME] }}
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Run TPC-H benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_tpch
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
TEST_OLAP_SCALE: 10
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Periodic TPC-H perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
user-examples-compare:
|
|
# if: ${{ !cancelled() && (github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null) }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
needs: [ generate-matrices, tpch-compare, prepare_AWS_RDS_databases ]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJSON(needs.generate-matrices.outputs.olap-compare-matrix) }}
|
|
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
PG_VERSION: ${{ matrix.pg_version }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: ghcr.io/neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
case "${PG_VERSION}" in
|
|
16)
|
|
CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }}
|
|
;;
|
|
17)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_USER_EXAMPLE_CONNSTR_PG17 }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unsupported PG_VERSION=${PG_VERSION} for PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_AURORA_CONNSTR }}
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_POSTGRES_CONNSTR }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Run user examples
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_user_examples
|
|
pg_version: ${{ env.PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Periodic TPC-H perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|