mirror of
https://github.com/neondatabase/neon.git
synced 2026-07-04 04:30:38 +00:00
1255 lines
58 KiB
YAML
1255 lines
58 KiB
YAML
name: Benchmarking
|
|
|
|
on:
|
|
# uncomment to run on push for debugging your PR
|
|
# push:
|
|
# branches: [ your branch ]
|
|
schedule:
|
|
# * is a special character in YAML so you have to quote this string
|
|
# ┌───────────── minute (0 - 59)
|
|
# │ ┌───────────── hour (0 - 23)
|
|
# │ │ ┌───────────── day of the month (1 - 31)
|
|
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
|
|
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
|
|
- cron: '0 3 * * *' # run once a day, timezone is utc
|
|
workflow_dispatch: # adds ability to run this manually
|
|
inputs:
|
|
region_id:
|
|
description: 'Project region id. If not set, the default region will be used'
|
|
required: false
|
|
default: 'aws-us-east-2'
|
|
save_perf_report:
|
|
type: boolean
|
|
description: 'Publish perf report. If not set, the report will be published only for the main branch'
|
|
required: false
|
|
collect_olap_explain:
|
|
type: boolean
|
|
description: 'Collect EXPLAIN ANALYZE for OLAP queries. If not set, EXPLAIN ANALYZE will not be collected'
|
|
required: false
|
|
default: false
|
|
collect_pg_stat_statements:
|
|
type: boolean
|
|
description: 'Collect pg_stat_statements for OLAP queries. If not set, pg_stat_statements will not be collected'
|
|
required: false
|
|
default: false
|
|
run_AWS_RDS_AND_AURORA:
|
|
type: boolean
|
|
description: 'AWS-RDS and AWS-AURORA normally only run on Saturday. Set this to true to run them on every workflow_dispatch'
|
|
required: false
|
|
default: false
|
|
run_only_ingest_tests:
|
|
type: boolean
|
|
description: 'Run ingest tests but no other tests. If not set, all tests including ingest tests will be run'
|
|
required: false
|
|
default: false
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash -euxo pipefail {0}
|
|
|
|
concurrency:
|
|
# Allow only one workflow per any non-`main` branch.
|
|
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
ingest:
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
env:
|
|
PG_CONFIG: /tmp/neon/pg_install/v16/bin/pg_config
|
|
PSQL: /tmp/neon/pg_install/v16/bin/psql
|
|
PG_16_LIB_PATH: /tmp/neon/pg_install/v16/lib
|
|
PGCOPYDB: /pgcopydb/bin/pgcopydb
|
|
PGCOPYDB_LIB_PATH: /pgcopydb/lib
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
timeout-minutes: 1440
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials # necessary on Azure runners
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours is currently max associated with IAM role
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Create Neon Project
|
|
id: create-neon-project-ingest-target
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
region_id: aws-us-east-2
|
|
postgres_version: 16
|
|
compute_units: '[12, 12]' # we want to test large compute here to avoid compute-side bottleneck
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Initialize Neon project and retrieve current backpressure seconds
|
|
env:
|
|
NEW_PROJECT_CONNSTR: ${{ steps.create-neon-project-ingest-target.outputs.dsn }}
|
|
NEW_PROJECT_ID: ${{ steps.create-neon-project-ingest-target.outputs.project_id }}
|
|
run: |
|
|
echo "Initializing Neon project with project_id: ${NEW_PROJECT_ID}"
|
|
export LD_LIBRARY_PATH=${PG_16_LIB_PATH}
|
|
${PSQL} "${NEW_PROJECT_CONNSTR}" -c "CREATE EXTENSION neon; CREATE EXTENSION neon_utils;"
|
|
BACKPRESSURE_TIME_BEFORE_INGEST=$(${PSQL} "${NEW_PROJECT_CONNSTR}" -t -c "select backpressure_throttling_time()/1000000;")
|
|
echo "BACKPRESSURE_TIME_BEFORE_INGEST=${BACKPRESSURE_TIME_BEFORE_INGEST}" >> $GITHUB_ENV
|
|
|
|
- name: Create pgcopydb filter file
|
|
run: |
|
|
cat << EOF > /tmp/pgcopydb_filter.txt
|
|
[include-only-table]
|
|
public.events
|
|
public.emails
|
|
public.email_transmissions
|
|
public.payments
|
|
public.editions
|
|
public.edition_modules
|
|
public.sp_content
|
|
public.email_broadcasts
|
|
public.user_collections
|
|
public.devices
|
|
public.user_accounts
|
|
public.lessons
|
|
public.lesson_users
|
|
public.payment_methods
|
|
public.orders
|
|
public.course_emails
|
|
public.modules
|
|
public.users
|
|
public.module_users
|
|
public.courses
|
|
public.payment_gateway_keys
|
|
public.accounts
|
|
public.roles
|
|
public.payment_gateways
|
|
public.management
|
|
public.event_names
|
|
EOF
|
|
|
|
# - name: Invoke pgcopydb
|
|
# env:
|
|
# NEW_PROJECT_CONNSTR: ${{ steps.create-neon-project-ingest-target.outputs.dsn }}
|
|
# BENCHMARK_INGEST_SOURCE_CONNSTR: ${{ secrets.BENCHMARK_INGEST_SOURCE_CONNSTR }}
|
|
# run: |
|
|
# export LD_LIBRARY_PATH=${PGCOPYDB_LIB_PATH}:${PG_16_LIB_PATH}
|
|
# export PGCOPYDB_SOURCE_PGURI="${BENCHMARK_INGEST_SOURCE_CONNSTR}"
|
|
# export PGCOPYDB_TARGET_PGURI="${NEW_PROJECT_CONNSTR}"
|
|
# ${PG_CONFIG} --bindir
|
|
# ${PGCOPYDB} --version
|
|
# ${PGCOPYDB} clone --skip-vacuum --no-owner --no-acl --skip-db-properties --table-jobs 4 \
|
|
# --index-jobs 4 --restore-jobs 4 --split-tables-larger-than 10GB --skip-extensions \
|
|
# --use-copy-binary --filters /tmp/pgcopydb_filter.txt 2>&1 | tee /tmp/pgcopydb_new_project.log
|
|
|
|
# create dummy pgcopydb log to test parsing
|
|
- name: create dummy log for parser test
|
|
run: |
|
|
cat << EOF > /tmp/pgcopydb_new_project.log
|
|
2024-11-04 18:00:53.433 500861 INFO main.c:136 Running pgcopydb version 0.17.10.g8361a93 from "/usr/lib/postgresql/17/bin/pgcopydb"
|
|
2024-11-04 18:00:53.434 500861 INFO cli_common.c:1225 [SOURCE] Copying database from "postgres://neondb_owner@ep-bitter-shape-w2c1ir0a.us-east-2.aws.neon.build/neondb?sslmode=require&keepalives=1&keepalives_idle=10&keepalives_interval=10&keepalives_count=60"
|
|
2024-11-04 18:00:53.434 500861 INFO cli_common.c:1226 [TARGET] Copying database into "postgres://neondb_owner@ep-icy-union-w25qd5pj.us-east-2.aws.neon.build/ludicrous?sslmode=require&keepalives=1&keepalives_idle=10&keepalives_interval=10&keepalives_count=60"
|
|
2024-11-04 18:00:53.442 500861 INFO copydb.c:105 Using work dir "/tmp/pgcopydb"
|
|
2024-11-04 18:00:53.541 500861 INFO snapshot.c:107 Exported snapshot "00000008-00000033-1" from the source database
|
|
2024-11-04 18:00:53.556 500865 INFO cli_clone_follow.c:543 STEP 1: fetch source database tables, indexes, and sequences
|
|
2024-11-04 18:00:54.570 500865 INFO copydb_schema.c:716 Splitting source candidate tables larger than 10 GB
|
|
2024-11-04 18:00:54.570 500865 INFO copydb_schema.c:829 Table public.events is 96 GB large which is larger than --split-tables-larger-than 10 GB, and does not have a unique column of type integer: splitting by CTID
|
|
2024-11-04 18:01:05.538 500865 INFO copydb_schema.c:905 Table public.events is 96 GB large, 10 COPY processes will be used, partitioning on ctid.
|
|
2024-11-04 18:01:05.564 500865 INFO copydb_schema.c:905 Table public.email_transmissions is 27 GB large, 4 COPY processes will be used, partitioning on id.
|
|
2024-11-04 18:01:05.584 500865 INFO copydb_schema.c:905 Table public.lessons is 25 GB large, 4 COPY processes will be used, partitioning on id.
|
|
2024-11-04 18:01:05.605 500865 INFO copydb_schema.c:905 Table public.lesson_users is 16 GB large, 3 COPY processes will be used, partitioning on id.
|
|
2024-11-04 18:01:05.605 500865 INFO copydb_schema.c:761 Fetched information for 26 tables (including 4 tables split in 21 partitions total), with an estimated total of 907 million tuples and 175 GB on-disk
|
|
2024-11-04 18:01:05.687 500865 INFO copydb_schema.c:968 Fetched information for 57 indexes (supporting 25 constraints)
|
|
2024-11-04 18:01:05.753 500865 INFO sequences.c:78 Fetching information for 24 sequences
|
|
2024-11-04 18:01:05.903 500865 INFO copydb_schema.c:1122 Fetched information for 4 extensions
|
|
2024-11-04 18:01:06.178 500865 INFO copydb_schema.c:1538 Found 0 indexes (supporting 0 constraints) in the target database
|
|
2024-11-04 18:01:06.184 500865 INFO cli_clone_follow.c:584 STEP 2: dump the source database schema (pre/post data)
|
|
2024-11-04 18:01:06.186 500865 INFO pgcmd.c:468 /usr/lib/postgresql/16/bin/pg_dump -Fc --snapshot 00000008-00000033-1 --section=pre-data --section=post-data --file /tmp/pgcopydb/schema/schema.dump 'postgres://neondb_owner@ep-bitter-shape-w2c1ir0a.us-east-2.aws.neon.build/neondb?sslmode=require&keepalives=1&keepalives_idle=10&keepalives_interval=10&keepalives_count=60'
|
|
2024-11-04 18:01:06.952 500865 INFO cli_clone_follow.c:592 STEP 3: restore the pre-data section to the target database
|
|
2024-11-04 18:01:07.004 500865 INFO pgcmd.c:1001 /usr/lib/postgresql/16/bin/pg_restore --dbname 'postgres://neondb_owner@ep-icy-union-w25qd5pj.us-east-2.aws.neon.build/ludicrous?sslmode=require&keepalives=1&keepalives_idle=10&keepalives_interval=10&keepalives_count=60' --section pre-data --jobs 4 --no-owner --no-acl --use-list /tmp/pgcopydb/schema/pre-filtered.list /tmp/pgcopydb/schema/schema.dump
|
|
2024-11-04 18:01:07.438 500874 INFO table-data.c:656 STEP 4: starting 4 table-data COPY processes
|
|
2024-11-04 18:01:07.451 500877 INFO vacuum.c:139 STEP 8: skipping VACUUM jobs per --skip-vacuum
|
|
2024-11-04 18:01:07.457 500875 INFO indexes.c:182 STEP 6: starting 4 CREATE INDEX processes
|
|
2024-11-04 18:01:07.457 500875 INFO indexes.c:183 STEP 7: constraints are built by the CREATE INDEX processes
|
|
2024-11-04 18:01:07.507 500865 INFO blobs.c:74 Skipping large objects: none found.
|
|
2024-11-04 18:01:07.509 500865 INFO sequences.c:194 STEP 9: reset sequences values
|
|
2024-11-04 18:01:07.510 500886 INFO sequences.c:290 Set sequences values on the target database
|
|
2024-11-04 20:49:00.587 500865 INFO cli_clone_follow.c:608 STEP 10: restore the post-data section to the target database
|
|
2024-11-04 20:49:00.600 500865 INFO pgcmd.c:1001 /usr/lib/postgresql/16/bin/pg_restore --dbname 'postgres://neondb_owner@ep-icy-union-w25qd5pj.us-east-2.aws.neon.build/ludicrous?sslmode=require&keepalives=1&keepalives_idle=10&keepalives_interval=10&keepalives_count=60' --section post-data --jobs 4 --no-owner --no-acl --use-list /tmp/pgcopydb/schema/post-filtered.list /tmp/pgcopydb/schema/schema.dump
|
|
2024-11-05 10:50:58.508 500865 INFO cli_clone_follow.c:639 All step are now done, 16h49m elapsed
|
|
2024-11-05 10:50:58.508 500865 INFO summary.c:3155 Printing summary for 26 tables and 57 indexes
|
|
|
|
OID | Schema | Name | Parts | copy duration | transmitted bytes | indexes | create index duration
|
|
------+--------+----------------------+-------+---------------+-------------------+---------+----------------------
|
|
24654 | public | events | 10 | 1d11h | 878 GB | 1 | 1h41m
|
|
24623 | public | email_transmissions | 4 | 4h46m | 99 GB | 3 | 2h04m
|
|
24665 | public | lessons | 4 | 4h42m | 161 GB | 4 | 1m11s
|
|
24661 | public | lesson_users | 3 | 2h46m | 49 GB | 3 | 39m35s
|
|
24631 | public | emails | 1 | 34m07s | 10 GB | 2 | 17s
|
|
24739 | public | payments | 1 | 5m47s | 1848 MB | 4 | 4m40s
|
|
24681 | public | module_users | 1 | 4m57s | 1610 MB | 3 | 1m50s
|
|
24694 | public | orders | 1 | 2m50s | 835 MB | 3 | 1m05s
|
|
24597 | public | devices | 1 | 1m45s | 498 MB | 2 | 40s
|
|
24723 | public | payment_methods | 1 | 1m24s | 548 MB | 2 | 31s
|
|
24765 | public | user_collections | 1 | 2m17s | 1005 MB | 2 | 968ms
|
|
24774 | public | users | 1 | 52s | 291 MB | 4 | 27s
|
|
24760 | public | user_accounts | 1 | 16s | 172 MB | 3 | 16s
|
|
24606 | public | edition_modules | 1 | 8s983 | 46 MB | 3 | 4s749
|
|
24583 | public | course_emails | 1 | 8s526 | 26 MB | 2 | 996ms
|
|
24685 | public | modules | 1 | 1s592 | 21 MB | 3 | 1s696
|
|
24610 | public | editions | 1 | 2s199 | 7483 kB | 2 | 1s032
|
|
24755 | public | sp_content | 1 | 1s555 | 4177 kB | 0 | 0ms
|
|
24619 | public | email_broadcasts | 1 | 744ms | 2645 kB | 2 | 677ms
|
|
24590 | public | courses | 1 | 387ms | 1540 kB | 2 | 367ms
|
|
24704 | public | payment_gateway_keys | 1 | 1s972 | 164 kB | 2 | 27ms
|
|
24576 | public | accounts | 1 | 58ms | 24 kB | 1 | 14ms
|
|
24647 | public | event_names | 1 | 32ms | 397 B | 1 | 8ms
|
|
24716 | public | payment_gateways | 1 | 1s675 | 117 B | 1 | 11ms
|
|
24748 | public | roles | 1 | 71ms | 173 B | 1 | 8ms
|
|
24676 | public | management | 1 | 33ms | 40 B | 1 | 19ms
|
|
|
|
|
|
Step Connection Duration Transfer Concurrency
|
|
-------------------------------------------------- ---------- ---------- ---------- ------------
|
|
Catalog Queries (table ordering, filtering, etc) source 12s 1
|
|
Dump Schema source 765ms 1
|
|
Prepare Schema target 466ms 1
|
|
COPY, INDEX, CONSTRAINTS, VACUUM (wall clock) both 2h47m 12
|
|
COPY (cumulative) both 7h46m 1225 GB 4
|
|
CREATE INDEX (cumulative) target 4h36m 4
|
|
CONSTRAINTS (cumulative) target 8s493 4
|
|
VACUUM (cumulative) target 0ms 4
|
|
Reset Sequences both 60ms 1
|
|
Large Objects (cumulative) (null) 0ms 0
|
|
Finalize Schema both 14h01m 4
|
|
-------------------------------------------------- ---------- ---------- ---------- ------------
|
|
Total Wall Clock Duration both 16h49m 20
|
|
|
|
|
|
EOF
|
|
|
|
|
|
- name: show tables sizes and retrieve current backpressure seconds
|
|
env:
|
|
NEW_PROJECT_CONNSTR: ${{ steps.create-neon-project-ingest-target.outputs.dsn }}
|
|
run: |
|
|
export LD_LIBRARY_PATH=${PG_16_LIB_PATH}
|
|
${PSQL} "${NEW_PROJECT_CONNSTR}" -c "\dt+"
|
|
BACKPRESSURE_TIME_AFTER_INGEST=$(${PSQL} "${NEW_PROJECT_CONNSTR}" -t -c "select backpressure_throttling_time()/1000000;")
|
|
echo "BACKPRESSURE_TIME_AFTER_INGEST=${BACKPRESSURE_TIME_AFTER_INGEST}" >> $GITHUB_ENV
|
|
|
|
- name: Parse pgcopydb log and report performance metrics
|
|
env:
|
|
PERF_TEST_RESULT_CONNSTR: ${{ secrets.PERF_TEST_RESULT_CONNSTR }}
|
|
run: |
|
|
export LD_LIBRARY_PATH=${PG_16_LIB_PATH}
|
|
|
|
# Define the log file path
|
|
LOG_FILE="/tmp/pgcopydb_new_project.log"
|
|
|
|
# Get the current git commit hash
|
|
git config --global --add safe.directory /__w/neon/neon
|
|
COMMIT_HASH=$(git rev-parse --short HEAD)
|
|
|
|
# Define the platform and test suite
|
|
PLATFORM="pg16-new-empty-project-us-east-2-staging"
|
|
SUIT="pgcopydb_ingest_bench"
|
|
|
|
# Function to convert time (e.g., "2h47m", "4h36m", "118ms", "8s493") to seconds
|
|
convert_to_seconds() {
|
|
local duration=$1
|
|
local total_seconds=0
|
|
|
|
# Check for hours (h)
|
|
if [[ "$duration" =~ ([0-9]+)h ]]; then
|
|
total_seconds=$((total_seconds + ${BASH_REMATCH[1]} * 3600))
|
|
fi
|
|
|
|
# Check for minutes (m)
|
|
if [[ "$duration" =~ ([0-9]+)m ]]; then
|
|
total_seconds=$((total_seconds + ${BASH_REMATCH[1]} * 60))
|
|
fi
|
|
|
|
# Check for seconds (s)
|
|
if [[ "$duration" =~ ([0-9]+)s ]]; then
|
|
total_seconds=$((total_seconds + ${BASH_REMATCH[1]}))
|
|
fi
|
|
|
|
# Check for milliseconds (ms) (if applicable)
|
|
if [[ "$duration" =~ ([0-9]+)ms ]]; then
|
|
total_seconds=$((total_seconds + ${BASH_REMATCH[1]} / 1000))
|
|
fi
|
|
|
|
echo $total_seconds
|
|
}
|
|
|
|
# Calculate the backpressure difference in seconds
|
|
BACKPRESSURE_TIME_DIFF=$(awk "BEGIN {print $BACKPRESSURE_TIME_AFTER_INGEST - $BACKPRESSURE_TIME_BEFORE_INGEST}")
|
|
|
|
# Insert the backpressure time difference into the performance database
|
|
if [ ! -z "$BACKPRESSURE_TIME_DIFF" ]; then
|
|
PSQL_CMD="${PSQL} \"${PERF_TEST_RESULT_CONNSTR}\" -c \"
|
|
INSERT INTO public.perf_test_results (suit, revision, platform, metric_name, metric_value, metric_unit, metric_report_type, recorded_at_timestamp)
|
|
VALUES ('${SUIT}', '${COMMIT_HASH}', '${PLATFORM}', 'backpressure_time', ${BACKPRESSURE_TIME_DIFF}, 'seconds', 'lower_is_better', now());
|
|
\""
|
|
echo "Inserting backpressure time difference: ${BACKPRESSURE_TIME_DIFF} seconds"
|
|
eval $PSQL_CMD
|
|
fi
|
|
|
|
# Extract and process log lines
|
|
while IFS= read -r line; do
|
|
METRIC_NAME=""
|
|
# Match each desired line and extract the relevant information
|
|
if [[ "$line" =~ COPY,\ INDEX,\ CONSTRAINTS,\ VACUUM.* ]]; then
|
|
METRIC_NAME="COPY, INDEX, CONSTRAINTS, VACUUM (wall clock)"
|
|
elif [[ "$line" =~ COPY\ \(cumulative\).* ]]; then
|
|
METRIC_NAME="COPY (cumulative)"
|
|
elif [[ "$line" =~ CREATE\ INDEX\ \(cumulative\).* ]]; then
|
|
METRIC_NAME="CREATE INDEX (cumulative)"
|
|
elif [[ "$line" =~ CONSTRAINTS\ \(cumulative\).* ]]; then
|
|
METRIC_NAME="CONSTRAINTS (cumulative)"
|
|
elif [[ "$line" =~ Finalize\ Schema.* ]]; then
|
|
METRIC_NAME="Finalize Schema"
|
|
elif [[ "$line" =~ Total\ Wall\ Clock\ Duration.* ]]; then
|
|
METRIC_NAME="Total Wall Clock Duration"
|
|
fi
|
|
|
|
# If a metric was matched, insert it into the performance database
|
|
if [ ! -z "$METRIC_NAME" ]; then
|
|
METRIC_VALUE=$(convert_to_seconds "$(echo $line | awk '{print $3}')")
|
|
PSQL_CMD="${PSQL} \"${PERF_TEST_RESULT_CONNSTR}\" -c \"
|
|
INSERT INTO public.perf_test_results (suit, revision, platform, metric_name, metric_value, metric_unit, metric_report_type, recorded_at_timestamp)
|
|
VALUES ('${SUIT}', '${COMMIT_HASH}', '${PLATFORM}', '${METRIC_NAME}', ${METRIC_VALUE}, 'seconds', 'lower_is_better', now());
|
|
\""
|
|
echo "Inserting ${METRIC_NAME} with value ${METRIC_VALUE} seconds"
|
|
eval $PSQL_CMD
|
|
fi
|
|
done < "$LOG_FILE"
|
|
|
|
- name: Delete Neon Project
|
|
if: ${{ always() }}
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project-ingest-target.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
bench:
|
|
if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- DEFAULT_PG_VERSION: 16
|
|
PLATFORM: "neon-staging"
|
|
region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
|
|
RUNNER: [ self-hosted, us-east-2, x64 ]
|
|
- DEFAULT_PG_VERSION: 16
|
|
PLATFORM: "azure-staging"
|
|
region_id: 'azure-eastus2'
|
|
RUNNER: [ self-hosted, eastus2, x64 ]
|
|
env:
|
|
TEST_PG_BENCH_DURATIONS_MATRIX: "300"
|
|
TEST_PG_BENCH_SCALES_MATRIX: "10,100"
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: ${{ matrix.DEFAULT_PG_VERSION }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.PLATFORM }}
|
|
|
|
runs-on: ${{ matrix.RUNNER }}
|
|
container:
|
|
image: neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials # necessary on Azure runners
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Create Neon Project
|
|
id: create-neon-project
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
region_id: ${{ matrix.region_id }}
|
|
postgres_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Run benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
# Set --sparse-ordering option of pytest-order plugin
|
|
# to ensure tests are running in order of appears in the file.
|
|
# It's important for test_perf_pgbench.py::test_pgbench_remote_* tests
|
|
extra_params:
|
|
-m remote_cluster
|
|
--sparse-ordering
|
|
--timeout 14400
|
|
--ignore test_runner/performance/test_perf_olap.py
|
|
--ignore test_runner/performance/test_perf_pgvector_queries.py
|
|
--ignore test_runner/performance/test_logical_replication.py
|
|
--ignore test_runner/performance/test_physical_replication.py
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Delete Neon Project
|
|
if: ${{ always() }}
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C033QLM5P7D" # dev-staging-stream
|
|
slack-message: |
|
|
Periodic perf testing: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
replication-tests:
|
|
if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 16
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: "neon-staging"
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Run Logical Replication benchmarks
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_logical_replication.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 5400
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
BENCHMARK_PROJECT_ID_PUB: ${{ vars.BENCHMARK_PROJECT_ID_PUB }}
|
|
BENCHMARK_PROJECT_ID_SUB: ${{ vars.BENCHMARK_PROJECT_ID_SUB }}
|
|
|
|
- name: Run Physical Replication benchmarks
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_physical_replication.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 5400
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
store-test-results-into-db: true
|
|
env:
|
|
REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C06T9AMNDQQ" # on-call-compute-staging-stream
|
|
slack-message: |
|
|
Periodic replication testing: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
generate-matrices:
|
|
if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
|
|
# Create matrices for the benchmarking jobs, so we run benchmarks on rds only once a week (on Saturday)
|
|
#
|
|
# Available platforms:
|
|
# - neonvm-captest-new: Freshly created project (1 CU)
|
|
# - neonvm-captest-freetier: Use freetier-sized compute (0.25 CU)
|
|
# - neonvm-captest-azure-new: Freshly created project (1 CU) in azure region
|
|
# - neonvm-captest-azure-freetier: Use freetier-sized compute (0.25 CU) in azure region
|
|
# - neonvm-captest-reuse: Reusing existing project
|
|
# - rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
|
|
# - rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
|
|
env:
|
|
RUN_AWS_RDS_AND_AURORA: ${{ github.event.inputs.run_AWS_RDS_AND_AURORA || 'false' }}
|
|
DEFAULT_REGION_ID: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
|
|
runs-on: ubuntu-22.04
|
|
outputs:
|
|
pgbench-compare-matrix: ${{ steps.pgbench-compare-matrix.outputs.matrix }}
|
|
olap-compare-matrix: ${{ steps.olap-compare-matrix.outputs.matrix }}
|
|
tpch-compare-matrix: ${{ steps.tpch-compare-matrix.outputs.matrix }}
|
|
|
|
steps:
|
|
- name: Generate matrix for pgbench benchmark
|
|
id: pgbench-compare-matrix
|
|
run: |
|
|
region_id_default=${{ env.DEFAULT_REGION_ID }}
|
|
runner_default='["self-hosted", "us-east-2", "x64"]'
|
|
runner_azure='["self-hosted", "eastus2", "x64"]'
|
|
image_default="neondatabase/build-tools:pinned-bookworm"
|
|
matrix='{
|
|
"pg_version" : [
|
|
16
|
|
],
|
|
"region_id" : [
|
|
"'"$region_id_default"'"
|
|
],
|
|
"platform": [
|
|
"neonvm-captest-new",
|
|
"neonvm-captest-reuse",
|
|
"neonvm-captest-new"
|
|
],
|
|
"db_size": [ "10gb" ],
|
|
"runner": ['"$runner_default"'],
|
|
"image": [ "'"$image_default"'" ],
|
|
"include": [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-new", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-freetier", "db_size": "3gb" ,"runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" },
|
|
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "10gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" },
|
|
{ "pg_version": 16, "region_id": "azure-eastus2", "platform": "neonvm-azure-captest-new", "db_size": "50gb","runner": '"$runner_azure"', "image": "neondatabase/build-tools:pinned-bookworm" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "neonvm-captest-sharding-reuse", "db_size": "50gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]
|
|
}'
|
|
|
|
if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
|
|
matrix=$(echo "$matrix" | jq '.include += [{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "rds-postgres", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" },
|
|
{ "pg_version": 16, "region_id": "'"$region_id_default"'", "platform": "rds-aurora", "db_size": "10gb","runner": '"$runner_default"', "image": "'"$image_default"'" }]')
|
|
fi
|
|
|
|
echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Generate matrix for OLAP benchmarks
|
|
id: olap-compare-matrix
|
|
run: |
|
|
matrix='{
|
|
"platform": [
|
|
"neonvm-captest-reuse"
|
|
]
|
|
}'
|
|
|
|
if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
|
|
matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres" },
|
|
{ "platform": "rds-aurora" }]')
|
|
fi
|
|
|
|
echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Generate matrix for TPC-H benchmarks
|
|
id: tpch-compare-matrix
|
|
run: |
|
|
matrix='{
|
|
"platform": [
|
|
"neonvm-captest-reuse"
|
|
],
|
|
"scale": [
|
|
"10"
|
|
]
|
|
}'
|
|
|
|
if [ "$(date +%A)" = "Saturday" ] || [ ${RUN_AWS_RDS_AND_AURORA} = "true" ]; then
|
|
matrix=$(echo "$matrix" | jq '.include += [{ "platform": "rds-postgres", "scale": "10" },
|
|
{ "platform": "rds-aurora", "scale": "10" }]')
|
|
fi
|
|
|
|
echo "matrix=$(echo "$matrix" | jq --compact-output '.')" >> $GITHUB_OUTPUT
|
|
|
|
prepare_AWS_RDS_databases:
|
|
if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
|
|
uses: ./.github/workflows/_benchmarking_preparation.yml
|
|
secrets: inherit
|
|
|
|
pgbench-compare:
|
|
if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
|
|
needs: [ generate-matrices, prepare_AWS_RDS_databases ]
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{fromJson(needs.generate-matrices.outputs.pgbench-compare-matrix)}}
|
|
|
|
env:
|
|
TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
|
|
TEST_PG_BENCH_SCALES_MATRIX: ${{ matrix.db_size }}
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: ${{ matrix.pg_version }}
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: ${{ matrix.runner }}
|
|
container:
|
|
image: ${{ matrix.image }}
|
|
options: --init
|
|
|
|
# Increase timeout to 8h, default timeout is 6h
|
|
timeout-minutes: 480
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Create Neon Project
|
|
if: contains(fromJson('["neonvm-captest-new", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform)
|
|
id: create-neon-project
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
region_id: ${{ matrix.region_id }}
|
|
postgres_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
compute_units: ${{ (contains(matrix.platform, 'captest-freetier') && '[0.25, 0.25]') || '[1, 1]' }}
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CONNSTR }}
|
|
;;
|
|
neonvm-captest-sharding-reuse)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_SHARDING_CONNSTR }}
|
|
;;
|
|
neonvm-captest-new | neonvm-captest-freetier | neonvm-azure-captest-new | neonvm-azure-captest-freetier)
|
|
CONNSTR=${{ steps.create-neon-project.outputs.dsn }}
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CONNSTR }}
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CONNSTR }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Benchmark init
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_init
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Benchmark simple-update
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_simple_update
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Benchmark select-only
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgbench_remote_select_only
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Delete Neon Project
|
|
if: ${{ steps.create-neon-project.outputs.project_id && always() }}
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C033QLM5P7D" # dev-staging-stream
|
|
slack-message: |
|
|
Periodic perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
pgbench-pgvector:
|
|
if: ${{ github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- PLATFORM: "neonvm-captest-pgvector"
|
|
RUNNER: [ self-hosted, us-east-2, x64 ]
|
|
- PLATFORM: "azure-captest-pgvector"
|
|
RUNNER: [ self-hosted, eastus2, x64 ]
|
|
|
|
env:
|
|
TEST_PG_BENCH_DURATIONS_MATRIX: "15m"
|
|
TEST_PG_BENCH_SCALES_MATRIX: "1"
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 16
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.PLATFORM }}
|
|
|
|
runs-on: ${{ matrix.RUNNER }}
|
|
container:
|
|
image: neondatabase/build-tools:pinned
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
# until https://github.com/neondatabase/neon/issues/8275 is fixed we temporarily install postgresql-16
|
|
# instead of using Neon artifacts containing pgbench
|
|
- name: Install postgresql-16 where pytest expects it
|
|
run: |
|
|
# Just to make it easier to test things locally on macOS (with arm64)
|
|
arch=$(uname -m | sed 's/x86_64/amd64/g' | sed 's/aarch64/arm64/g')
|
|
|
|
cd /home/nonroot
|
|
wget -q "https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-17/libpq5_17.0-1.pgdg110+1_${arch}.deb"
|
|
wget -q "https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/postgresql-client-16_16.4-1.pgdg110+2_${arch}.deb"
|
|
wget -q "https://apt.postgresql.org/pub/repos/apt/pool/main/p/postgresql-16/postgresql-16_16.4-1.pgdg110+2_${arch}.deb"
|
|
dpkg -x libpq5_17.0-1.pgdg110+1_${arch}.deb pg
|
|
dpkg -x postgresql-16_16.4-1.pgdg110+2_${arch}.deb pg
|
|
dpkg -x postgresql-client-16_16.4-1.pgdg110+2_${arch}.deb pg
|
|
|
|
mkdir -p /tmp/neon/pg_install/v16/bin
|
|
ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/pgbench /tmp/neon/pg_install/v16/bin/pgbench
|
|
ln -s /home/nonroot/pg/usr/lib/postgresql/16/bin/psql /tmp/neon/pg_install/v16/bin/psql
|
|
ln -s /home/nonroot/pg/usr/lib/$(uname -m)-linux-gnu /tmp/neon/pg_install/v16/lib
|
|
|
|
LD_LIBRARY_PATH="/home/nonroot/pg/usr/lib/$(uname -m)-linux-gnu:${LD_LIBRARY_PATH:-}"
|
|
export LD_LIBRARY_PATH
|
|
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> ${GITHUB_ENV}
|
|
|
|
/tmp/neon/pg_install/v16/bin/pgbench --version
|
|
/tmp/neon/pg_install/v16/bin/psql --version
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-pgvector)
|
|
CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR }}
|
|
;;
|
|
azure-captest-pgvector)
|
|
CONNSTR=${{ secrets.BENCHMARK_PGVECTOR_CONNSTR_AZURE }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Benchmark pgvector hnsw indexing
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_pgvector_indexing
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
|
|
- name: Benchmark pgvector queries
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_pgvector_queries.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C033QLM5P7D" # dev-staging-stream
|
|
slack-message: |
|
|
Periodic perf testing on ${{ env.PLATFORM }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
clickbench-compare:
|
|
# ClichBench DB for rds-aurora and rds-Postgres deployed to the same clusters
|
|
# we use for performance testing in pgbench-compare.
|
|
# Run this job only when pgbench-compare is finished to avoid the intersection.
|
|
# We might change it after https://github.com/neondatabase/neon/issues/2900.
|
|
#
|
|
# *_CLICKBENCH_CONNSTR: Genuine ClickBench DB with ~100M rows
|
|
# *_CLICKBENCH_10M_CONNSTR: DB with the first 10M rows of ClickBench DB
|
|
if: ${{ !cancelled() && (github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null) }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
needs: [ generate-matrices, pgbench-compare, prepare_AWS_RDS_databases ]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }}
|
|
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 16
|
|
TEST_OUTPUT: /tmp/test_output
|
|
TEST_OLAP_COLLECT_EXPLAIN: ${{ github.event.inputs.collect_olap_explain }}
|
|
TEST_OLAP_COLLECT_PG_STAT_STATEMENTS: ${{ github.event.inputs.collect_pg_stat_statements }}
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
|
|
# Increase timeout to 12h, default timeout is 6h
|
|
# we have regression in clickbench causing it to run 2-3x longer
|
|
timeout-minutes: 720
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
CONNSTR=${{ secrets.BENCHMARK_CAPTEST_CLICKBENCH_10M_CONNSTR }}
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_AURORA_CLICKBENCH_10M_CONNSTR }}
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR=${{ secrets.BENCHMARK_RDS_POSTGRES_CLICKBENCH_10M_CONNSTR }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: ClickBench benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 43200 -k test_clickbench
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
TEST_OLAP_COLLECT_EXPLAIN: ${{ github.event.inputs.collect_olap_explain || 'false' }}
|
|
TEST_OLAP_COLLECT_PG_STAT_STATEMENTS: ${{ github.event.inputs.collect_pg_stat_statements || 'false' }}
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
TEST_OLAP_SCALE: 10
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C033QLM5P7D" # dev-staging-stream
|
|
slack-message: |
|
|
Periodic OLAP perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
tpch-compare:
|
|
# TCP-H DB for rds-aurora and rds-Postgres deployed to the same clusters
|
|
# we use for performance testing in pgbench-compare & clickbench-compare.
|
|
# Run this job only when clickbench-compare is finished to avoid the intersection.
|
|
# We might change it after https://github.com/neondatabase/neon/issues/2900.
|
|
#
|
|
# *_TPCH_S10_CONNSTR: DB generated with scale factor 10 (~10 GB)
|
|
if: ${{ !cancelled() && (github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null) }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
needs: [ generate-matrices, clickbench-compare, prepare_AWS_RDS_databases ]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJson(needs.generate-matrices.outputs.tpch-compare-matrix) }}
|
|
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 16
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
TEST_OLAP_SCALE: ${{ matrix.scale }}
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Get Connstring Secret Name
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
ENV_PLATFORM=CAPTEST_TPCH
|
|
;;
|
|
rds-aurora)
|
|
ENV_PLATFORM=RDS_AURORA_TPCH
|
|
;;
|
|
rds-postgres)
|
|
ENV_PLATFORM=RDS_POSTGRES_TPCH
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
CONNSTR_SECRET_NAME="BENCHMARK_${ENV_PLATFORM}_S${TEST_OLAP_SCALE}_CONNSTR"
|
|
echo "CONNSTR_SECRET_NAME=${CONNSTR_SECRET_NAME}" >> $GITHUB_ENV
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
CONNSTR=${{ secrets[env.CONNSTR_SECRET_NAME] }}
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Run TPC-H benchmark
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_tpch
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
TEST_OLAP_SCALE: ${{ matrix.scale }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C033QLM5P7D" # dev-staging-stream
|
|
slack-message: |
|
|
Periodic TPC-H perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
user-examples-compare:
|
|
if: ${{ !cancelled() && (github.event.inputs.run_only_ingest_tests == 'false' || github.event.inputs.run_only_ingest_tests == null) }}
|
|
permissions:
|
|
contents: write
|
|
statuses: write
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
needs: [ generate-matrices, tpch-compare, prepare_AWS_RDS_databases ]
|
|
|
|
strategy:
|
|
fail-fast: false
|
|
matrix: ${{ fromJson(needs.generate-matrices.outputs.olap-compare-matrix) }}
|
|
|
|
env:
|
|
POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
|
|
DEFAULT_PG_VERSION: 16
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: remote
|
|
SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
|
|
PLATFORM: ${{ matrix.platform }}
|
|
|
|
runs-on: [ self-hosted, us-east-2, x64 ]
|
|
container:
|
|
image: neondatabase/build-tools:pinned-bookworm
|
|
credentials:
|
|
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
|
|
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
|
|
options: --init
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@v4
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
role-duration-seconds: 18000 # 5 hours
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
|
|
- name: Set up Connection String
|
|
id: set-up-connstr
|
|
run: |
|
|
case "${PLATFORM}" in
|
|
neonvm-captest-reuse)
|
|
CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_CAPTEST_CONNSTR }}
|
|
;;
|
|
rds-aurora)
|
|
CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_AURORA_CONNSTR }}
|
|
;;
|
|
rds-postgres)
|
|
CONNSTR=${{ secrets.BENCHMARK_USER_EXAMPLE_RDS_POSTGRES_CONNSTR }}
|
|
;;
|
|
*)
|
|
echo >&2 "Unknown PLATFORM=${PLATFORM}. Allowed only 'neonvm-captest-reuse', 'rds-aurora', or 'rds-postgres'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
|
|
echo "connstr=${CONNSTR}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Run user examples
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: ${{ env.BUILD_TYPE }}
|
|
test_selection: performance/test_perf_olap.py
|
|
run_in_parallel: false
|
|
save_perf_report: ${{ env.SAVE_PERF_REPORT }}
|
|
extra_params: -m remote_cluster --timeout 21600 -k test_user_examples
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
env:
|
|
VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
|
|
PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
|
|
BENCHMARK_CONNSTR: ${{ steps.set-up-connstr.outputs.connstr }}
|
|
|
|
- name: Create Allure report
|
|
id: create-allure-report
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-generate
|
|
|
|
- name: Post to a Slack channel
|
|
if: ${{ github.event.schedule && failure() }}
|
|
uses: slackapi/slack-github-action@v1
|
|
with:
|
|
channel-id: "C033QLM5P7D" # dev-staging-stream
|
|
slack-message: |
|
|
Periodic TPC-H perf testing on ${{ matrix.platform }}: ${{ job.status }}
|
|
<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>
|
|
<${{ steps.create-allure-report.outputs.report-url }}|Allure report>
|
|
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|