neon/.github/actions/run-python-test-set/action.yml

name: 'Run python test'
description: 'Runs a Neon python test set, performing all the required preparations before'

inputs:
  build_type:
    description: 'Type of Rust (neon) and C (postgres) builds. Must be "release" or "debug", or "remote" for the remote cluster'
    required: true
  test_selection:
    description: 'A python test suite to run'
    required: true
  extra_params:
    description: 'Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr'
    required: false
    default: ''
  needs_postgres_source:
    description: 'Set to true if the test suite requires postgres source checked out'
    required: false
    default: 'false'
  run_in_parallel:
    description: 'Whether to run tests in parallel'
    required: false
    default: 'true'
  save_perf_report:
    description: 'Whether to upload the performance report, if true PERF_TEST_RESULT_CONNSTR env variable should be set'
    required: false
    default: 'false'
  run_with_real_s3:
    description: 'Whether to pass real s3 credentials to the test suite'
    required: false
    default: 'false'
  real_s3_bucket:
    description: 'Bucket name for real s3 tests'
    required: false
    default: ''
  real_s3_region:
    description: 'Region name for real s3 tests'
    required: false
    default: ''
  rerun_failed:
    description: 'Whether to rerun failed tests'
    required: false
    default: 'false'
  pg_version:
    description: 'Postgres version to use for tests'
    required: false
    default: 'v16'
  sanitizers:
    description: 'enabled or disabled'
    required: false
    default: 'disabled'
    type: string
  benchmark_durations:
    description: 'benchmark durations JSON'
    required: false
    default: '{}'
  aws-oicd-role-arn:
    description: 'OIDC role arn to interract with S3'
    required: true

runs:
  using: "composite"
  steps:
    - name: Get Neon artifact
      if: inputs.build_type != 'remote'
      uses: ./.github/actions/download
      with:
        name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build_type }}${{ inputs.sanitizers == 'enabled' && '-sanitized' || '' }}-artifact
        path: /tmp/neon
        aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}

    - name: Download Neon binaries for the previous release
      if: inputs.build_type != 'remote'
      uses: ./.github/actions/download
      with:
        name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build_type }}-artifact
        path: /tmp/neon-previous
        prefix: latest
        aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}

    - name: Download compatibility snapshot
      if: inputs.build_type != 'remote'
      uses: ./.github/actions/download
      with:
        name: compatibility-snapshot-${{ runner.arch }}-${{ inputs.build_type }}-pg${{ inputs.pg_version }}
        path: /tmp/compatibility_snapshot_pg${{ inputs.pg_version }}
        prefix: latest
        # The lack of compatibility snapshot (for example, for the new Postgres version)
        # shouldn't fail the whole job. Only relevant test should fail.
        skip-if-does-not-exist: true
        aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}

    - name: Checkout
      if: inputs.needs_postgres_source == 'true'
      uses: actions/checkout@v4
      with:
        submodules: true

    - name: Cache poetry deps
      uses: actions/cache@v4
      with:
        path: ~/.cache/pypoetry/virtualenvs
        key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }}

    - name: Install Python deps
      shell: bash -euxo pipefail {0}
      run: ./scripts/pysync

    - name: Run pytest
      env:
        NEON_BIN: /tmp/neon/bin
        COMPATIBILITY_NEON_BIN: /tmp/neon-previous/bin
        COMPATIBILITY_POSTGRES_DISTRIB_DIR: /tmp/neon-previous/pg_install
        TEST_OUTPUT: /tmp/test_output
        BUILD_TYPE: ${{ inputs.build_type }}
        COMPATIBILITY_SNAPSHOT_DIR: /tmp/compatibility_snapshot_pg${{ inputs.pg_version }}
        RERUN_FAILED: ${{ inputs.rerun_failed }}
        PG_VERSION: ${{ inputs.pg_version }}
        SANITIZERS: ${{ inputs.sanitizers }}
      shell: bash -euxo pipefail {0}
      run: |
        # PLATFORM will be embedded in the perf test report
        # and it is needed to distinguish different environments
        export PLATFORM=${PLATFORM:-github-actions-selfhosted}
        export POSTGRES_DISTRIB_DIR=${POSTGRES_DISTRIB_DIR:-/tmp/neon/pg_install}
        export DEFAULT_PG_VERSION=${PG_VERSION#v}
        export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib
        export BENCHMARK_CONNSTR=${BENCHMARK_CONNSTR:-}
        export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0:abort_on_error=1:strict_string_checks=1:check_initialization_order=1:strict_init_order=1
        export UBSAN_OPTIONS=abort_on_error=1:print_stacktrace=1

        if [ "${BUILD_TYPE}" = "remote" ]; then
          export REMOTE_ENV=1
        fi

        PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)"
        rm -rf $PERF_REPORT_DIR

        TEST_SELECTION="test_runner/${{ inputs.test_selection }}"
        EXTRA_PARAMS="${{ inputs.extra_params }}"
        if [ -z "$TEST_SELECTION" ]; then
          echo "test_selection must be set"
          exit 1
        fi
        if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then
          # -n sets the number of parallel processes that pytest-xdist will run
          EXTRA_PARAMS="-n12 $EXTRA_PARAMS"

          # --dist=loadgroup points tests marked with @pytest.mark.xdist_group
          # to the same worker to make @pytest.mark.order work with xdist
          EXTRA_PARAMS="--dist=loadgroup $EXTRA_PARAMS"
        fi

        if [[ "${{ inputs.run_with_real_s3 }}" == "true" ]]; then
          echo "REAL S3 ENABLED"
          export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
          export REMOTE_STORAGE_S3_BUCKET=${{ inputs.real_s3_bucket }}
          export REMOTE_STORAGE_S3_REGION=${{ inputs.real_s3_region }}
        fi

        if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then
          mkdir -p "$PERF_REPORT_DIR"
          EXTRA_PARAMS="--out-dir $PERF_REPORT_DIR $EXTRA_PARAMS"
        fi

        if [ "${RERUN_FAILED}" == "true" ]; then
          EXTRA_PARAMS="--reruns 2 $EXTRA_PARAMS"
        fi

        # We use pytest-split plugin to run benchmarks in parallel on different CI runners
        if [ "${TEST_SELECTION}" = "test_runner/performance" ] && [ "${{ inputs.build_type }}" != "remote" ]; then
          mkdir -p $TEST_OUTPUT
          echo '${{ inputs.benchmark_durations || '{}' }}' > $TEST_OUTPUT/benchmark_durations.json

          EXTRA_PARAMS="--durations-path $TEST_OUTPUT/benchmark_durations.json $EXTRA_PARAMS"
        fi

        if [[ $BUILD_TYPE == "debug" && $RUNNER_ARCH == 'X64' ]]; then
          cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
        else
          cov_prefix=()
        fi

        # Wake up the cluster if we use remote neon instance
        if [ "${{ inputs.build_type }}" = "remote" ] && [ -n "${BENCHMARK_CONNSTR}" ]; then
          QUERIES=("SELECT version()")
          if [[ "${PLATFORM}" = "neon"* ]]; then
            QUERIES+=("SHOW neon.tenant_id")
            QUERIES+=("SHOW neon.timeline_id")
          fi

          for q in "${QUERIES[@]}"; do
            ${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/psql ${BENCHMARK_CONNSTR} -c "${q}"
          done
        fi

        # Run the tests.
        #
        # --alluredir saves test results in Allure format (in a specified directory)
        # --verbose prints name of each test (helpful when there are
        # multiple tests in one file)
        # -rA prints summary in the end
        # -s is not used to prevent pytest from capturing output, because tests are running
        # in parallel and logs are mixed between different tests
        #
        mkdir -p $TEST_OUTPUT/allure/results
        "${cov_prefix[@]}" ./scripts/pytest \
          --alluredir=$TEST_OUTPUT/allure/results \
          --tb=short \
          --verbose \
          -rA $TEST_SELECTION $EXTRA_PARAMS

        if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then
          export REPORT_FROM="$PERF_REPORT_DIR"
          export REPORT_TO="$PLATFORM"
          scripts/generate_and_push_perf_report.sh
        fi

    - name: Upload compatibility snapshot
      # Note, that we use `github.base_ref` which is a target branch for a PR
      if: github.event_name == 'pull_request' && github.base_ref == 'release'
      uses: ./.github/actions/upload
      with:
        name: compatibility-snapshot-${{ runner.arch }}-${{ inputs.build_type }}-pg${{ inputs.pg_version }}
        # Directory is created by test_compatibility.py::test_create_snapshot, keep the path in sync with the test
        path: /tmp/test_output/compatibility_snapshot_pg${{ inputs.pg_version }}/
        # The lack of compatibility snapshot shouldn't fail the job
        # (for example if we didn't run the test for non build-and-test workflow)
        skip-if-does-not-exist: true
        aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}

    - uses: aws-actions/configure-aws-credentials@v4
      if: ${{ !cancelled() }}
      with:
        aws-region: eu-central-1
        role-to-assume: ${{ inputs.aws-oicd-role-arn }}
        role-duration-seconds: 3600 # 1 hour should be more than enough to upload report

    - name: Upload test results
      if: ${{ !cancelled() }}
      uses: ./.github/actions/allure-report-store
      with:
        report-dir: /tmp/test_output/allure/results
        unique-key: ${{ inputs.build_type }}-${{ inputs.pg_version }}-${{ runner.arch }}
        aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}