neon/.github/workflows/pg-clients.yml

name: Test Postgres client libraries

on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    #          ┌───────────── minute (0 - 59)
    #          │ ┌───────────── hour (0 - 23)
    #          │ │ ┌───────────── day of the month (1 - 31)
    #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
    #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
    - cron:  '23 02 * * *' # run once a day, timezone is utc
  pull_request:
    paths:
      - '.github/workflows/pg-clients.yml'
      - 'test_runner/pg_clients/**/*.py'
      - 'test_runner/logical_repl/**/*.py'
      - 'poetry.lock'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

defaults:
  run:
    shell: bash -euxo pipefail {0}

permissions:
  id-token: write # aws-actions/configure-aws-credentials
  statuses: write # require for posting a status update

env:
  DEFAULT_PG_VERSION: 17
  PLATFORM: neon-captest-new
  AWS_DEFAULT_REGION: eu-central-1

jobs:
  check-permissions:
    if: ${{ !contains(github.event.pull_request.labels.*.name, 'run-no-ci') }}
    uses: ./.github/workflows/check-permissions.yml
    with:
      github-event-name: ${{ github.event_name }}

  build-build-tools-image:
    permissions:
      packages: write
    needs: [ check-permissions ]
    uses: ./.github/workflows/build-build-tools-image.yml
    secrets: inherit

  generate-ch-tmppw:
    runs-on: ubuntu-22.04
    outputs:
      tmp_val: ${{ steps.pwgen.outputs.tmp_val }}
    steps:
      - name: Generate a random password
        id: pwgen
        run: |
          set +x
          p=$(dd if=/dev/random bs=14 count=1 2>/dev/null | base64)
          echo tmp_val="${p//\//}" >> "${GITHUB_OUTPUT}"

  test-logical-replication:
    needs: [ build-build-tools-image, generate-ch-tmppw ]
    runs-on: ubuntu-22.04

    container:
      image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --init --user root
    services:
      clickhouse:
        image: clickhouse/clickhouse-server:24.8
        env:
          CLICKHOUSE_PASSWORD: ${{ needs.generate-ch-tmppw.outputs.tmp_val }}
        ports:
          - 9000:9000
          - 8123:8123
      zookeeper:
        image: quay.io/debezium/zookeeper:3.1.3.Final
        ports:
          - 2181:2181
          - 2888:2888
          - 3888:3888
      kafka:
        image: quay.io/debezium/kafka:3.1.3.Final
        env:
          ZOOKEEPER_CONNECT: "zookeeper:2181"
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
          KAFKA_BROKER_ID: 1
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_JMX_PORT: 9991
        ports:
          - 9092:9092
      debezium:
        image: quay.io/debezium/connect:3.1.3.Final
        env:
          BOOTSTRAP_SERVERS: kafka:9092
          GROUP_ID: 1
          CONFIG_STORAGE_TOPIC: debezium-config
          OFFSET_STORAGE_TOPIC: debezium-offset
          STATUS_STORAGE_TOPIC: debezium-status
          DEBEZIUM_CONFIG_CONNECTOR_CLASS: io.debezium.connector.postgresql.PostgresConnector
        ports:
          - 8083:8083
    steps:
      - name: Harden the runner (Audit all outbound calls)
        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
        with:
          egress-policy: audit

      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

      - name: Download Neon artifact
        uses: ./.github/actions/download
        with:
          name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
          path: /tmp/neon/
          prefix: latest
          aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}

      - name: Create Neon Project
        id: create-neon-project
        uses: ./.github/actions/neon-project-create
        with:
          api_key: ${{ secrets.NEON_STAGING_API_KEY }}
          postgres_version: ${{ env.DEFAULT_PG_VERSION }}
          project_settings: >-
            {"enable_logical_replication": true}

      - name: Run tests
        uses: ./.github/actions/run-python-test-set
        with:
          build_type: remote
          test_selection: logical_repl
          run_in_parallel: false
          extra_params: -m remote_cluster
          pg_version: ${{ env.DEFAULT_PG_VERSION }}
          aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
        env:
          BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
          CLICKHOUSE_PASSWORD: ${{ needs.generate-ch-tmppw.outputs.tmp_val }}

      - name: Delete Neon Project
        if: always()
        uses: ./.github/actions/neon-project-delete
        with:
          project_id: ${{ steps.create-neon-project.outputs.project_id }}
          api_key: ${{ secrets.NEON_STAGING_API_KEY }}

      - name: Create Allure report
        if: ${{ !cancelled() }}
        id: create-allure-report
        uses: ./.github/actions/allure-report-generate
        with:
          store-test-results-into-db: true
          aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
        env:
          REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}

      - name: Post to a Slack channel
        if: github.event.schedule && failure()
        uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
        with:
          channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
          slack-message: |
            Testing the logical replication: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ job.status }}> (<${{ steps.create-allure-report.outputs.report-url }}|test report>)
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

  test-postgres-client-libs:
    needs: [ build-build-tools-image ]
    runs-on: ubuntu-22.04

    container:
      image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --init --user root

    steps:
    - name: Harden the runner (Audit all outbound calls)
      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
      with:
        egress-policy: audit

    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

    - name: Download Neon artifact
      uses: ./.github/actions/download
      with:
        name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
        path: /tmp/neon/
        prefix: latest
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}

    - name: Create Neon Project
      id: create-neon-project
      uses: ./.github/actions/neon-project-create
      with:
        api_key: ${{ secrets.NEON_STAGING_API_KEY }}
        postgres_version: ${{ env.DEFAULT_PG_VERSION }}

    - name: Run tests
      uses: ./.github/actions/run-python-test-set
      with:
        build_type: remote
        test_selection: pg_clients
        run_in_parallel: false
        extra_params: -m remote_cluster
        pg_version: ${{ env.DEFAULT_PG_VERSION }}
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
      env:
        BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}

    - name: Delete Neon Project
      if: always()
      uses: ./.github/actions/neon-project-delete
      with:
        project_id: ${{ steps.create-neon-project.outputs.project_id }}
        api_key: ${{ secrets.NEON_STAGING_API_KEY }}

    - name: Create Allure report
      if: ${{ !cancelled() }}
      id: create-allure-report
      uses: ./.github/actions/allure-report-generate
      with:
        store-test-results-into-db: true
        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
      env:
        REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}

    - name: Post to a Slack channel
      if: github.event.schedule && failure()
      uses: slackapi/slack-github-action@fcfb566f8b0aab22203f066d80ca1d7e4b5d05b3 # v1.27.1
      with:
        channel-id: "C06KHQVQ7U3" # on-call-qa-staging-stream
        slack-message: |
          Testing Postgres clients: <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ job.status }}> (<${{ steps.create-allure-report.outputs.report-url }}|test report>)
      env:
        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}