mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
1ed7252950
## Problem We used ClickHouse v. 24.8, which is outdated, for logical replication testing. We could miss some problems. ## Summary of changes The version was updated to 25.6, with a workaround using the environment variable `PGSSLCERT`. Co-authored-by: Alexey Masterov <alexey.masterov@databricks.com>
246 lines
8.6 KiB
YAML
246 lines
8.6 KiB
YAML
name: Test Postgres client libraries
|
|
|
|
on:
|
|
schedule:
|
|
# * is a special character in YAML so you have to quote this string
|
|
# ┌───────────── minute (0 - 59)
|
|
# │ ┌───────────── hour (0 - 23)
|
|
# │ │ ┌───────────── day of the month (1 - 31)
|
|
# │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
|
|
# │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
|
|
- cron: '23 02 * * *' # run once a day, timezone is utc
|
|
pull_request:
|
|
paths:
|
|
- '.github/workflows/pg-clients.yml'
|
|
- 'test_runner/pg_clients/**/*.py'
|
|
- 'test_runner/logical_repl/**/*.py'
|
|
- 'poetry.lock'
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref_name }}
|
|
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
|
|
|
defaults:
|
|
run:
|
|
shell: bash -euxo pipefail {0}
|
|
|
|
permissions:
|
|
id-token: write # aws-actions/configure-aws-credentials
|
|
statuses: write # require for posting a status update
|
|
|
|
env:
|
|
DEFAULT_PG_VERSION: 17
|
|
PLATFORM: neon-captest-new
|
|
AWS_DEFAULT_REGION: eu-central-1
|
|
|
|
jobs:
|
|
check-permissions:
|
|
if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
|
|
uses: ./.github/workflows/check-permissions.yml
|
|
with:
|
|
github-event-name: ${{ github.event_name }}
|
|
|
|
build-build-tools-image:
|
|
permissions:
|
|
packages: write
|
|
needs: [ check-permissions ]
|
|
uses: ./.github/workflows/build-build-tools-image.yml
|
|
secrets: inherit
|
|
|
|
generate-ch-tmppw:
|
|
runs-on: ubuntu-22.04
|
|
outputs:
|
|
tmp_val: ${{ steps.pwgen.outputs.tmp_val }}
|
|
steps:
|
|
- name: Generate a random password
|
|
id: pwgen
|
|
run: |
|
|
set +x
|
|
p=$(dd if=/dev/random bs=14 count=1 2>/dev/null | base64)
|
|
echo tmp_val="${p//\//}" >> "${GITHUB_OUTPUT}"
|
|
|
|
test-logical-replication:
|
|
needs: [ build-build-tools-image, generate-ch-tmppw ]
|
|
runs-on: ubuntu-22.04
|
|
|
|
container:
|
|
image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init --user root
|
|
services:
|
|
clickhouse:
|
|
image: clickhouse/clickhouse-server:25.6
|
|
env:
|
|
CLICKHOUSE_PASSWORD: ${{ needs.generate-ch-tmppw.outputs.tmp_val }}
|
|
PGSSLCERT: /tmp/postgresql.crt
|
|
ports:
|
|
- 9000:9000
|
|
- 8123:8123
|
|
zookeeper:
|
|
image: quay.io/debezium/zookeeper:3.1.3.Final
|
|
ports:
|
|
- 2181:2181
|
|
- 2888:2888
|
|
- 3888:3888
|
|
kafka:
|
|
image: quay.io/debezium/kafka:3.1.3.Final
|
|
env:
|
|
ZOOKEEPER_CONNECT: "zookeeper:2181"
|
|
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
|
|
KAFKA_BROKER_ID: 1
|
|
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
|
|
KAFKA_JMX_PORT: 9991
|
|
ports:
|
|
- 9092:9092
|
|
debezium:
|
|
image: quay.io/debezium/connect:3.1.3.Final
|
|
env:
|
|
BOOTSTRAP_SERVERS: kafka:9092
|
|
GROUP_ID: 1
|
|
CONFIG_STORAGE_TOPIC: debezium-config
|
|
OFFSET_STORAGE_TOPIC: debezium-offset
|
|
STATUS_STORAGE_TOPIC: debezium-status
|
|
DEBEZIUM_CONFIG_CONNECTOR_CLASS: io.debezium.connector.postgresql.PostgresConnector
|
|
ports:
|
|
- 8083:8083
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Create Neon Project
|
|
id: create-neon-project
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
postgres_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
project_settings: >-
|
|
{"enable_logical_replication": true}
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: remote
|
|
test_selection: logical_repl
|
|
run_in_parallel: false
|
|
extra_params: -m remote_cluster
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
|
|
CLICKHOUSE_PASSWORD: ${{ needs.generate-ch-tmppw.outputs.tmp_val }}
|
|
|
|
- name: Delete Neon Project
|
|
if: always()
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
if: ${{ !cancelled() }}
|
|
id: create-allure-report
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
store-test-results-into-db: true
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: github.event.schedule && failure()
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Testing the logical replication: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ job.status }}> (<${{ steps.create-allure-report.outputs.report-url }}|test report>)
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|
|
|
|
test-postgres-client-libs:
|
|
needs: [ build-build-tools-image ]
|
|
runs-on: ubuntu-22.04
|
|
|
|
container:
|
|
image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
|
|
credentials:
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
options: --init --user root
|
|
|
|
steps:
|
|
- name: Harden the runner (Audit all outbound calls)
|
|
uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
|
|
|
|
- name: Download Neon artifact
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
|
|
path: /tmp/neon/
|
|
prefix: latest
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
|
|
- name: Create Neon Project
|
|
id: create-neon-project
|
|
uses: ./.github/actions/neon-project-create
|
|
with:
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
postgres_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
|
|
- name: Run tests
|
|
uses: ./.github/actions/run-python-test-set
|
|
with:
|
|
build_type: remote
|
|
test_selection: pg_clients
|
|
run_in_parallel: false
|
|
extra_params: -m remote_cluster
|
|
pg_version: ${{ env.DEFAULT_PG_VERSION }}
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
|
|
|
|
- name: Delete Neon Project
|
|
if: always()
|
|
uses: ./.github/actions/neon-project-delete
|
|
with:
|
|
project_id: ${{ steps.create-neon-project.outputs.project_id }}
|
|
api_key: ${{ secrets.NEON_STAGING_API_KEY }}
|
|
|
|
- name: Create Allure report
|
|
if: ${{ !cancelled() }}
|
|
id: create-allure-report
|
|
uses: ./.github/actions/allure-report-generate
|
|
with:
|
|
store-test-results-into-db: true
|
|
aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
|
|
env:
|
|
REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
|
|
|
|
- name: Post to a Slack channel
|
|
if: github.event.schedule && failure()
|
|
uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
|
|
with:
|
|
channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
|
|
slack-message: |
|
|
Testing Postgres clients: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ job.status }}> (<${{ steps.create-allure-report.outputs.report-url }}|test report>)
|
|
env:
|
|
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
|