From 9747e90f3ac4a5c126c134d1ea07f7699923d59c Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 8 Dec 2022 22:28:25 +0000 Subject: [PATCH] Nightly Benchmarks: Move from captest to staging (#2838) Migrate Nightly Benchmarks from captest to staging. - Migrate GitHub Workflows - Replace `zenith-benchmarker` with regular runners - Remove `environment` parameter from Neon GitHub Actions, add `postgres_version` - The only job left on captest is `neon-captest-reuse`, which will be moved to staging after its project migration. Ref https://github.com/neondatabase/cloud/issues/2836 --- .github/actions/neon-branch-create/action.yml | 40 ++--- .github/actions/neon-branch-delete/action.yml | 29 +--- .../actions/neon-project-create/action.yml | 43 ++--- .../actions/neon-project-delete/action.yml | 29 +--- .github/workflows/benchmarking.yml | 156 +++++++----------- .github/workflows/pg_clients.yml | 6 +- 6 files changed, 97 insertions(+), 206 deletions(-) diff --git a/.github/actions/neon-branch-create/action.yml b/.github/actions/neon-branch-create/action.yml index 9102bcf3da..7ee43a3587 100644 --- a/.github/actions/neon-branch-create/action.yml +++ b/.github/actions/neon-branch-create/action.yml @@ -5,12 +5,12 @@ inputs: api_key: desctiption: 'Neon API key' required: true - environment: - desctiption: 'dev (aka captest) or staging' - required: true project_id: desctiption: 'ID of the Project to create Branch in' required: true + api_host: + desctiption: 'Neon API host' + default: console.stage.neon.tech outputs: dsn: description: 'Created Branch DSN (for main database)' @@ -22,27 +22,6 @@ outputs: runs: using: "composite" steps: - - name: Parse Input - id: parse-input - shell: bash -euxo pipefail {0} - run: | - case "${ENVIRONMENT}" in - dev) - API_HOST=console.dev.neon.tech - ;; - staging) - API_HOST=console.stage.neon.tech - ;; - *) - echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only" - exit 1 - ;; - esac - - echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT - env: - ENVIRONMENT: ${{ inputs.environment }} - - name: Create New Branch id: create-branch shell: bash -euxo pipefail {0} @@ -56,7 +35,12 @@ runs: --data "{ \"branch\": { \"name\": \"Created by actions/neon-branch-create; GITHUB_RUN_ID=${GITHUB_RUN_ID} at $(date +%s)\" - } + }, + \"endpoints\": [ + { + \"type\": \"read_write\" + } + ] }") if [ -z "${branch}" ]; then @@ -84,8 +68,8 @@ runs: host=$(echo $branch | jq --raw-output '.endpoints[0].host') echo "host=${host}" >> $GITHUB_OUTPUT env: + API_HOST: ${{ inputs.api_host }} API_KEY: ${{ inputs.api_key }} - API_HOST: ${{ steps.parse-input.outputs.api_host }} PROJECT_ID: ${{ inputs.project_id }} - name: Get Role name @@ -103,8 +87,8 @@ runs: role_name=$(echo $roles | jq --raw-output '.roles[] | select(.protected == false) | .name') echo "role_name=${role_name}" >> $GITHUB_OUTPUT env: + API_HOST: ${{ inputs.api_host }} API_KEY: ${{ inputs.api_key }} - API_HOST: ${{ steps.parse-input.outputs.api_host }} PROJECT_ID: ${{ inputs.project_id }} BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }} @@ -146,8 +130,8 @@ runs: echo "::add-mask::${dsn}" echo "dsn=${dsn}" >> $GITHUB_OUTPUT env: + API_HOST: ${{ inputs.api_host }} API_KEY: ${{ inputs.api_key }} - API_HOST: ${{ steps.parse-input.outputs.api_host }} PROJECT_ID: ${{ inputs.project_id }} BRANCH_ID: ${{ steps.create-branch.outputs.branch_id }} ROLE_NAME: ${{ steps.role-name.outputs.role_name }} diff --git a/.github/actions/neon-branch-delete/action.yml b/.github/actions/neon-branch-delete/action.yml index 3ca96ced11..5689093e2e 100644 --- a/.github/actions/neon-branch-delete/action.yml +++ b/.github/actions/neon-branch-delete/action.yml @@ -5,40 +5,19 @@ inputs: api_key: desctiption: 'Neon API key' required: true - environment: - desctiption: 'dev (aka captest) or staging' - required: true project_id: desctiption: 'ID of the Project which should be deleted' required: true branch_id: desctiption: 'ID of the branch to delete' required: true + api_host: + desctiption: 'Neon API host' + default: console.stage.neon.tech runs: using: "composite" steps: - - name: Parse Input - id: parse-input - shell: bash -euxo pipefail {0} - run: | - case "${ENVIRONMENT}" in - dev) - API_HOST=console.dev.neon.tech - ;; - staging) - API_HOST=console.stage.neon.tech - ;; - *) - echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only" - exit 1 - ;; - esac - - echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT - env: - ENVIRONMENT: ${{ inputs.environment }} - - name: Delete Branch # Do not try to delete a branch if .github/actions/neon-project-create # or .github/actions/neon-branch-create failed before @@ -73,7 +52,7 @@ runs: exit 1 fi env: + API_HOST: ${{ inputs.api_host }} API_KEY: ${{ inputs.api_key }} PROJECT_ID: ${{ inputs.project_id }} BRANCH_ID: ${{ inputs.branch_id }} - API_HOST: ${{ steps.parse-input.outputs.api_host }} diff --git a/.github/actions/neon-project-create/action.yml b/.github/actions/neon-project-create/action.yml index 8399d6c511..b9d9182882 100644 --- a/.github/actions/neon-project-create/action.yml +++ b/.github/actions/neon-project-create/action.yml @@ -5,12 +5,16 @@ inputs: api_key: desctiption: 'Neon API key' required: true - environment: - desctiption: 'dev (aka captest) or staging' - required: true region_id: desctiption: 'Region ID, if not set the project will be created in the default region' - required: false + default: aws-us-east-2 + postgres_version: + desctiption: 'Postgres version; default is 15' + default: 15 + api_host: + desctiption: 'Neon API host' + default: console.stage.neon.tech + outputs: dsn: description: 'Created Project DSN (for main database)' @@ -22,31 +26,6 @@ outputs: runs: using: "composite" steps: - - name: Parse Input - id: parse-input - shell: bash -euxo pipefail {0} - run: | - case "${ENVIRONMENT}" in - dev) - API_HOST=console.dev.neon.tech - REGION_ID=${REGION_ID:-aws-eu-west-1} - ;; - staging) - API_HOST=console.stage.neon.tech - REGION_ID=${REGION_ID:-aws-us-east-2} - ;; - *) - echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only" - exit 1 - ;; - esac - - echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT - echo "region_id=${REGION_ID}" >> $GITHUB_OUTPUT - env: - ENVIRONMENT: ${{ inputs.environment }} - REGION_ID: ${{ inputs.region_id }} - - name: Create Neon Project id: create-neon-project # A shell without `set -x` to not to expose password/dsn in logs @@ -61,6 +40,7 @@ runs: --data "{ \"project\": { \"name\": \"Created by actions/neon-project-create; GITHUB_RUN_ID=${GITHUB_RUN_ID}\", + \"pg_version\": ${POSTGRES_VERSION}, \"region_id\": \"${REGION_ID}\", \"settings\": { } } @@ -76,6 +56,7 @@ runs: project_id=$(echo $project | jq --raw-output '.project.id') echo "project_id=${project_id}" >> $GITHUB_OUTPUT env: + API_HOST: ${{ inputs.api_host }} API_KEY: ${{ inputs.api_key }} - API_HOST: ${{ steps.parse-input.outputs.api_host }} - REGION_ID: ${{ steps.parse-input.outputs.region_id }} + REGION_ID: ${{ inputs.region_id }} + POSTGRES_VERSION: ${{ inputs.postgres_version }} diff --git a/.github/actions/neon-project-delete/action.yml b/.github/actions/neon-project-delete/action.yml index 88b5d3fc5b..cd58c629e5 100644 --- a/.github/actions/neon-project-delete/action.yml +++ b/.github/actions/neon-project-delete/action.yml @@ -5,37 +5,16 @@ inputs: api_key: desctiption: 'Neon API key' required: true - environment: - desctiption: 'dev (aka captest) or staging' - required: true project_id: desctiption: 'ID of the Project to delete' required: true + api_host: + desctiption: 'Neon API host' + default: console.stage.neon.tech runs: using: "composite" steps: - - name: Parse Input - id: parse-input - shell: bash -euxo pipefail {0} - run: | - case "${ENVIRONMENT}" in - dev) - API_HOST=console.dev.neon.tech - ;; - staging) - API_HOST=console.stage.neon.tech - ;; - *) - echo 2>&1 "Unknown environment=${ENVIRONMENT}. Allowed 'dev' or 'staging' only" - exit 1 - ;; - esac - - echo "api_host=${API_HOST}" >> $GITHUB_OUTPUT - env: - ENVIRONMENT: ${{ inputs.environment }} - - name: Delete Neon Project # Do not try to delete a project if .github/actions/neon-project-create failed before if: ${{ inputs.project_id != '' }} @@ -49,6 +28,6 @@ runs: --header "Content-Type: application/json" \ --header "Authorization: Bearer ${API_KEY}" env: + API_HOST: ${{ inputs.api_host }} API_KEY: ${{ inputs.api_key }} PROJECT_ID: ${{ inputs.project_id }} - API_HOST: ${{ steps.parse-input.outputs.api_host }} diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index eb9ba70371..a2c05a9222 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -15,9 +15,6 @@ on: workflow_dispatch: # adds ability to run this manually inputs: - environment: - description: 'Environment to run remote tests on (dev or staging)' - required: false region_id: description: 'Use a particular region. If not set the default region will be used' required: false @@ -37,103 +34,69 @@ concurrency: jobs: bench: - # this workflow runs on self hosteed runner - # it's environment is quite different from usual guthub runner - # probably the most important difference is that it doesn't start from clean workspace each time - # e g if you install system packages they are not cleaned up since you install them directly in host machine - # not a container or something - # See documentation for more info: https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners - runs-on: [self-hosted, zenith-benchmarker] - env: - POSTGRES_DISTRIB_DIR: /usr/pgsql + TEST_PG_BENCH_DURATIONS_MATRIX: "300" + TEST_PG_BENCH_SCALES_MATRIX: "10,100" + POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install DEFAULT_PG_VERSION: 14 + TEST_OUTPUT: /tmp/test_output + BUILD_TYPE: remote + SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref == 'refs/heads/main' ) }} + PLATFORM: "neon-staging" + + runs-on: [ self-hosted, us-east-2, x64 ] + container: + image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned + options: --init steps: - - name: Checkout zenith repo - uses: actions/checkout@v3 + - uses: actions/checkout@v3 - # actions/setup-python@v2 is not working correctly on self-hosted runners - # see https://github.com/actions/setup-python/issues/162 - # and probably https://github.com/actions/setup-python/issues/162#issuecomment-865387976 in particular - # so the simplest solution to me is to use already installed system python and spin virtualenvs for job runs. - # there is Python 3.7.10 already installed on the machine so use it to install poetry and then use poetry's virtuealenvs - - name: Install poetry & deps - run: | - python3 -m pip install --upgrade poetry wheel - # since pip/poetry caches are reused there shouldn't be any troubles with install every time - ./scripts/pysync - - - name: Show versions - run: | - echo Python - python3 --version - poetry run python3 --version - echo Poetry - poetry --version - echo Pgbench - ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/pgbench --version + - name: Download Neon artifact + uses: ./.github/actions/download + with: + name: neon-${{ runner.os }}-release-artifact + path: /tmp/neon/ + prefix: latest - name: Create Neon Project id: create-neon-project uses: ./.github/actions/neon-project-create with: - environment: ${{ github.event.inputs.environment || 'staging' }} - api_key: ${{ ( github.event.inputs.environment || 'staging' ) == 'staging' && secrets.NEON_STAGING_API_KEY || secrets.NEON_CAPTEST_API_KEY }} + region_id: ${{ github.event.inputs.region_id || 'aws-us-east-2' }} + postgres_version: ${{ env.DEFAULT_PG_VERSION }} + api_key: ${{ secrets.NEON_STAGING_API_KEY }} - name: Run benchmark - # pgbench is installed system wide from official repo - # https://download.postgresql.org/pub/repos/yum/13/redhat/rhel-7-x86_64/ - # via - # sudo tee /etc/yum.repos.d/pgdg.repo<