Nightly Benchmarks: Move from captest to staging (#2838)

Migrate Nightly Benchmarks from captest to staging. - Migrate GitHub Workflows - Replace `zenith-benchmarker` with regular runners - Remove `environment` parameter from Neon GitHub Actions, add `postgres_version` - The only job left on captest is `neon-captest-reuse`, which will be moved to staging after its project migration. Ref https://github.com/neondatabase/cloud/issues/2836
2026-01-04 20:12:54 +00:00 · 2022-12-08 22:28:25 +00:00
parent a19c487766
commit 9747e90f3a
6 changed files with 97 additions and 206 deletions
--- a/.github/actions/neon-branch-create/action.yml
+++ b/.github/actions/neon-branch-create/action.yml
@@ -5,12 +5,12 @@ inputs:
  api_key:
    desctiption: 'Neon API key'
    required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
  project_id:
    desctiption: 'ID of the Project to create Branch in'
    required: true
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech
 outputs:
  dsn:
    description: 'Created Branch DSN (for main database)'
@@ -22,27 +22,6 @@ outputs:
 runs:
  using: "composite"
  steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-
    - name: Create New Branch
      id: create-branch
      shell: bash -euxo pipefail {0}
@@ -56,7 +35,12 @@ runs:
            --data "{
              \"branch\": {
                \"name\": \"Created by actions/neon-branch-create; GITHUB_RUN_ID=${GITHUB_RUN_ID} at $(date +%s)\"
-              }
+              },
+              \"endpoints\": [
+                {
+                  \"type\": \"read_write\"
+                }
+              ]
            }")

          if [ -z "${branch}" ]; then
@@ -84,8 +68,8 @@ runs:
        host=$(echo $branch | jq --raw-output '.endpoints[0].host')
        echo "host=${host}" >> $GITHUB_OUTPUT
      env:
+        API_HOST: ${{ inputs.api_host }}
        API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
        PROJECT_ID: ${{ inputs.project_id }}

    - name: Get Role name
@@ -103,8 +87,8 @@ runs:
        role_name=$(echo $roles | jq --raw-output '.roles[] | select(.protected == false) | .name')
        echo "role_name=${role_name}" >> $GITHUB_OUTPUT
      env:
+        API_HOST: ${{ inputs.api_host }}
        API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
        PROJECT_ID: ${{ inputs.project_id }}
        BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}

@@ -146,8 +130,8 @@ runs:
        echo "::add-mask::${dsn}"
        echo "dsn=${dsn}" >> $GITHUB_OUTPUT
      env:
+        API_HOST: ${{ inputs.api_host }}
        API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
        PROJECT_ID: ${{ inputs.project_id }}
        BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }}
        ROLE_NAME: ${{ steps.role-name.outputs.role_name }}
--- a/.github/actions/neon-branch-delete/action.yml
+++ b/.github/actions/neon-branch-delete/action.yml
@@ -5,40 +5,19 @@ inputs:
  api_key:
    desctiption: 'Neon API key'
    required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
  project_id:
    desctiption: 'ID of the Project which should be deleted'
    required: true
  branch_id:
    desctiption: 'ID of the branch to delete'
    required: true
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech

 runs:
  using: "composite"
  steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-
    - name: Delete Branch
      # Do not try to delete a branch if .github/actions/neon-project-create
      # or .github/actions/neon-branch-create failed before
@@ -73,7 +52,7 @@ runs:
          exit 1
        fi
      env:
+        API_HOST: ${{ inputs.api_host }}
        API_KEY: ${{ inputs.api_key }}
        PROJECT_ID: ${{ inputs.project_id }}
        BRANCH_ID: ${{ inputs.branch_id }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
--- a/.github/actions/neon-project-create/action.yml
+++ b/.github/actions/neon-project-create/action.yml
@@ -5,12 +5,16 @@ inputs:
  api_key:
    desctiption: 'Neon API key'
    required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
  region_id:
    desctiption: 'Region ID, if not set the project will be created in the default region'
-    required: false
+    default: aws-us-east-2
+  postgres_version:
+    desctiption: 'Postgres version; default is 15'
+    default: 15
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech
+
 outputs:
  dsn:
    description: 'Created Project DSN (for main database)'
@@ -22,31 +26,6 @@ outputs:
 runs:
  using: "composite"
  steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            REGION_ID=${REGION_ID:-aws-eu-west-1}
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            REGION_ID=${REGION_ID:-aws-us-east-2}
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-        echo "region_id=${REGION_ID}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-        REGION_ID: ${{ inputs.region_id }}
-
    - name: Create Neon Project
      id: create-neon-project
      # A shell without `set -x` to not to expose password/dsn in logs
@@ -61,6 +40,7 @@ runs:
          --data "{
            \"project\": {
              \"name\": \"Created by actions/neon-project-create; GITHUB_RUN_ID=${GITHUB_RUN_ID}\",
+              \"pg_version\": ${POSTGRES_VERSION},
              \"region_id\": \"${REGION_ID}\",
              \"settings\": { }
            }
@@ -76,6 +56,7 @@ runs:
        project_id=$(echo $project | jq --raw-output '.project.id')
        echo "project_id=${project_id}" >> $GITHUB_OUTPUT
      env:
+        API_HOST: ${{ inputs.api_host }}
        API_KEY: ${{ inputs.api_key }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
-        REGION_ID: ${{ steps.parse-input.outputs.region_id }}
+        REGION_ID: ${{ inputs.region_id }}
+        POSTGRES_VERSION: ${{ inputs.postgres_version }}
--- a/.github/actions/neon-project-delete/action.yml
+++ b/.github/actions/neon-project-delete/action.yml
@@ -5,37 +5,16 @@ inputs:
  api_key:
    desctiption: 'Neon API key'
    required: true
-  environment:
-    desctiption: 'dev (aka captest) or staging'
-    required: true
  project_id:
    desctiption: 'ID of the Project to delete'
    required: true
+  api_host:
+    desctiption: 'Neon API host'
+    default: console.stage.neon.tech

 runs:
  using: "composite"
  steps:
-    - name: Parse Input
-      id: parse-input
-      shell: bash -euxo pipefail {0}
-      run: |
-        case "${ENVIRONMENT}" in
-          dev)
-            API_HOST=console.dev.neon.tech
-            ;;
-          staging)
-            API_HOST=console.stage.neon.tech
-            ;;
-          *)
-            echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only"
-            exit 1
-            ;;
-        esac
-
-        echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT
-      env:
-        ENVIRONMENT: ${{ inputs.environment }}
-
    - name: Delete Neon Project
      # Do not try to delete a project if .github/actions/neon-project-create failed before
      if: ${{ inputs.project_id != '' }}
@@ -49,6 +28,6 @@ runs:
          --header "Content-Type: application/json" \
          --header "Authorization: Bearer ${API_KEY}"
      env:
+        API_HOST: ${{ inputs.api_host }}
        API_KEY: ${{ inputs.api_key }}
        PROJECT_ID: ${{ inputs.project_id }}
-        API_HOST: ${{ steps.parse-input.outputs.api_host }}
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -15,9 +15,6 @@ on:

  workflow_dispatch: # adds ability to run this manually
    inputs:
-      environment:
-        description: 'Environment to run remote tests on (dev or staging)'
-        required: false
      region_id:
        description: 'Use a particular region. If not set the default region will be used'
        required: false
@@ -37,103 +34,69 @@ concurrency:

 jobs:
  bench:
-    # this workflow runs on self hosteed runner
-    # it's environment is quite different from usual guthub runner
-    # probably the most important difference is that it doesn't start from clean workspace each time
-    # e g if you install system packages they are not cleaned up since you install them directly in host machine
-    # not a container or something
-    # See documentation for more info: https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners
-    runs-on: [self-hosted, zenith-benchmarker]
-
    env:
-      POSTGRES_DISTRIB_DIR: /usr/pgsql
+      TEST_PG_BENCH_DURATIONS_MATRIX: "300"
+      TEST_PG_BENCH_SCALES_MATRIX: "10,100"
+      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
      DEFAULT_PG_VERSION: 14
+      TEST_OUTPUT: /tmp/test_output
+      BUILD_TYPE: remote
+      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
+      PLATFORM: "neon-staging"
+
+    runs-on: [ self-hosted, us-east-2, x64 ]
+    container:
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
+      options: --init

    steps:
-    - name: Checkout zenith repo
-      uses: actions/checkout@v3
+    - uses: actions/checkout@v3

-    # actions/setup-python@v2 is not working correctly on self-hosted runners
-    # see https://github.com/actions/setup-python/issues/162
-    # and probably https://github.com/actions/setup-python/issues/162#issuecomment-865387976 in particular
-    # so the simplest solution to me is to use already installed system python and spin virtualenvs for job runs.
-    # there is Python 3.7.10 already installed on the machine so use it to install poetry and then use poetry's virtuealenvs
-    - name: Install poetry & deps
-      run: |
-        python3 -m pip install --upgrade poetry wheel
-        # since pip/poetry caches are reused there shouldn't be any troubles with install every time
-        ./scripts/pysync
-
-    - name: Show versions
-      run: |
-        echo Python
-        python3 --version
-        poetry run python3 --version
-        echo Poetry
-        poetry --version
-        echo Pgbench
-        ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version
+    - name: Download Neon artifact
+      uses: ./.github/actions/download
+      with:
+        name: neon-${{ runner.os }}-release-artifact
+        path: /tmp/neon/
+        prefix: latest

    - name: Create Neon Project
      id: create-neon-project
      uses: ./.github/actions/neon-project-create
      with:
-        environment: ${{ github.event.inputs.environment || 'staging' }}
-        api_key: ${{ ( github.event.inputs.environment || 'staging' ) == 'staging' && secrets.NEON_STAGING_API_KEY  || secrets.NEON_CAPTEST_API_KEY }}
+        region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
+        api_key: ${{ secrets.NEON_STAGING_API_KEY }}

    - name: Run benchmark
-      # pgbench is installed system wide from official repo
-      # https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/
-      # via
-      # sudo tee /etc/yum.repos.d/pgdg.repo<<EOF
-      # [pgdg13]
-      # name=PostgreSQL 13 for RHEL/CentOS 7 - x86_64
-      # baseurl=https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/
-      # enabled=1
-      # gpgcheck=0
-      # EOF
-      # sudo yum makecache
-      # sudo yum install postgresql13-contrib
-      # actual binaries are located in /usr/pgsql-13/bin/
+      uses: ./.github/actions/run-python-test-set
+      with:
+        build_type: ${{ env.BUILD_TYPE }}
+        test_selection: performance
+        run_in_parallel: false
+        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
+        # Set --sparse-ordering option of pytest-order plugin
+        # to ensure tests are running in order of appears in the file.
+        # It's important for test_perf_pgbench.py::test_pgbench_remote_* tests
+        extra_params: -m remote_cluster --sparse-ordering --timeout 5400 --ignore test_runner/performance/test_perf_olap.py
      env:
-        # The pgbench test runs two tests of given duration against each scale.
-        # So the total runtime with these parameters is 2 * 2 * 300 = 1200, or 20 minutes.
-        # Plus time needed to initialize the test databases.
-        TEST_PG_BENCH_DURATIONS_MATRIX: "300"
-        TEST_PG_BENCH_SCALES_MATRIX: "10,100"
-        PLATFORM: "neon-staging"
        BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
-        REMOTE_ENV: "1" # indicate to test harness that we do not have zenith binaries locally
-      run: |
-        # just to be sure that no data was cached on self hosted runner
-        # since it might generate duplicates when calling ingest_perf_test_result.py
-        rm -rf perf-report-staging
-        mkdir -p perf-report-staging
-        # Set --sparse-ordering option of pytest-order plugin to ensure tests are running in order of appears in the file,
-        # it's important for test_perf_pgbench.py::test_pgbench_remote_* tests.
-        # Do not run tests from test_runner/performance/test_perf_olap.py because they require a prepared DB. We run them separately in `clickbench-compare` job.
-        ./scripts/pytest test_runner/performance/ -v \
-          -m "remote_cluster" \
-          --sparse-ordering \
-          --out-dir perf-report-staging \
-          --timeout 5400 \
-          --ignore test_runner/performance/test_perf_olap.py
-
-    - name: Submit result
-      env:
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
-      run: |
-        REPORT_FROM=$(realpath perf-report-staging) REPORT_TO=staging scripts/generate_and_push_perf_report.sh

    - name: Delete Neon Project
      if: ${{ always() }}
      uses: ./.github/actions/neon-project-delete
      with:
-        environment: staging
        project_id: ${{ steps.create-neon-project.outputs.project_id }}
        api_key: ${{ secrets.NEON_STAGING_API_KEY }}

+    - name: Create Allure report
+      if: success() || failure()
+      uses: ./.github/actions/allure-report
+      with:
+        action: generate
+        build_type: ${{ env.BUILD_TYPE }}
+
    - name: Post to a Slack channel
      if: ${{ github.event.schedule && failure() }}
      uses: slackapi/slack-github-action@v1
@@ -152,15 +115,22 @@ jobs:
        # neon-captest-prefetch: Same, with prefetching enabled (new project)
        # rds-aurora: Aurora Postgres Serverless v2 with autoscaling from 0.5 to 2 ACUs
        # rds-postgres: RDS Postgres db.m5.large instance (2 vCPU, 8 GiB) with gp3 EBS storage
-        platform: [ neon-captest-new, neon-captest-reuse, neon-captest-prefetch, rds-postgres ]
+        platform: [ neon-captest-new, neon-captest-prefetch, rds-postgres ]
        db_size: [ 10gb ]
+        runner: [ us-east-2 ]
        include:
+          - platform: neon-captest-reuse
+            db_size: 10gb
+            runner: dev  # TODO: Switch to us-east-2 after dry-bonus-223539 migration to staging
          - platform: neon-captest-new
            db_size: 50gb
+            runner: us-east-2
          - platform: neon-captest-prefetch
            db_size: 50gb
+            runner: us-east-2
          - platform: rds-aurora
            db_size: 50gb
+            runner: us-east-2

    env:
      TEST_PG_BENCH_DURATIONS_MATRIX: "60m"
@@ -172,9 +142,9 @@ jobs:
      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
      PLATFORM: ${{ matrix.platform }}

-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: [ self-hosted, "${{ matrix.runner }}", x64 ]
    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init

    timeout-minutes: 360 # 6h
@@ -199,8 +169,9 @@ jobs:
      id: create-neon-project
      uses: ./.github/actions/neon-project-create
      with:
-        environment: ${{ github.event.inputs.environment || 'dev' }}
-        api_key: ${{ ( github.event.inputs.environment || 'dev' ) == 'staging' && secrets.NEON_STAGING_API_KEY  || secrets.NEON_CAPTEST_API_KEY }}
+        region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}
+        api_key: ${{ secrets.NEON_STAGING_API_KEY }}

    - name: Set up Connection String
      id: set-up-connstr
@@ -278,6 +249,13 @@ jobs:
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"

+    - name: Delete Neon Project
+      if: ${{ steps.create-neon-project.outputs.project_id && always() }}
+      uses: ./.github/actions/neon-project-delete
+      with:
+        project_id: ${{ steps.create-neon-project.outputs.project_id }}
+        api_key: ${{ secrets.NEON_STAGING_API_KEY }}
+
    - name: Create Allure report
      if: success() || failure()
      uses: ./.github/actions/allure-report
@@ -285,14 +263,6 @@ jobs:
        action: generate
        build_type: ${{ env.BUILD_TYPE }}

-    - name: Delete Neon Project
-      if: ${{ steps.create-neon-project.outputs.project_id && always() }}
-      uses: ./.github/actions/neon-project-delete
-      with:
-        environment: dev
-        project_id: ${{ steps.create-neon-project.outputs.project_id }}
-        api_key: ${{ secrets.NEON_CAPTEST_API_KEY }}
-
    - name: Post to a Slack channel
      if: ${{ github.event.schedule && failure() }}
      uses: slackapi/slack-github-action@v1
@@ -329,9 +299,9 @@ jobs:
      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
      PLATFORM: ${{ matrix.platform }}

-    runs-on: [ self-hosted, dev, x64 ]
+    runs-on: [ self-hosted, us-east-2, x64 ]
    container:
-      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
+      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned
      options: --init

    timeout-minutes: 360 # 6h
@@ -440,9 +410,7 @@ jobs:
      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }}
      PLATFORM: ${{ matrix.platform }}

-    # NOTE: Here we use non-standadard `captest` runner (instead of `dev`) which is located in us-east-2 region.
-    #       We will move the rest of benchmarking jobs to staging in https://github.com/neondatabase/neon/pull/2838
-    runs-on: [ self-hosted, captest, x64 ]
+    runs-on: [ self-hosted, us-east-2, x64 ]
    container:
      image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:pinned
      options: --init
--- a/.github/workflows/pg_clients.yml
+++ b/.github/workflows/pg_clients.yml
@@ -23,6 +23,7 @@ jobs:
    runs-on: [ ubuntu-latest ]

    env:
+      DEFAULT_PG_VERSION: 14
      TEST_OUTPUT: /tmp/test_output

    steps:
@@ -51,8 +52,8 @@ jobs:
      id: create-neon-project
      uses: ./.github/actions/neon-project-create
      with:
-        environment: staging
        api_key: ${{ secrets.NEON_STAGING_API_KEY }}
+        postgres_version: ${{ env.DEFAULT_PG_VERSION }}

    - name: Run pytest
      env:
@@ -63,7 +64,7 @@ jobs:
      run: |
        # Test framework expects we have psql binary;
        # but since we don't really need it in this test, let's mock it
-        mkdir -p "$POSTGRES_DISTRIB_DIR/v14/bin" && touch "$POSTGRES_DISTRIB_DIR/v14/bin/psql";
+        mkdir -p "$POSTGRES_DISTRIB_DIR/v${DEFAULT_PG_VERSION}/bin" && touch "$POSTGRES_DISTRIB_DIR/v${DEFAULT_PG_VERSION}/bin/psql";
        ./scripts/pytest \
          --junitxml=$TEST_OUTPUT/junit.xml \
          --tb=short \
@@ -75,7 +76,6 @@ jobs:
      if: ${{ always() }}
      uses: ./.github/actions/neon-project-delete
      with:
-        environment: staging
        project_id: ${{ steps.create-neon-project.outputs.project_id }}
        api_key: ${{ secrets.NEON_STAGING_API_KEY }}