mirror of
https://github.com/neondatabase/neon.git
synced 2025-12-22 21:59:59 +00:00
977781e423
Add a build with sanitizers (asan, ubsan) to the CI pipeline and run tests on it. See https://github.com/neondatabase/neon/issues/6053 --------- Co-authored-by: Alexander Bayandin <alexander@neon.tech>
241 lines
9.6 KiB
YAML
241 lines
9.6 KiB
YAML
name: 'Run python test'
|
|
description: 'Runs a Neon python test set, performing all the required preparations before'
|
|
|
|
inputs:
|
|
build_type:
|
|
description: 'Type of Rust (neon) and C (postgres) builds. Must be "release" or "debug", or "remote" for the remote cluster'
|
|
required: true
|
|
test_selection:
|
|
description: 'A python test suite to run'
|
|
required: true
|
|
extra_params:
|
|
description: 'Arbitrary parameters to pytest. For example "-s" to prevent capturing stdout/stderr'
|
|
required: false
|
|
default: ''
|
|
needs_postgres_source:
|
|
description: 'Set to true if the test suite requires postgres source checked out'
|
|
required: false
|
|
default: 'false'
|
|
run_in_parallel:
|
|
description: 'Whether to run tests in parallel'
|
|
required: false
|
|
default: 'true'
|
|
save_perf_report:
|
|
description: 'Whether to upload the performance report, if true PERF_TEST_RESULT_CONNSTR env variable should be set'
|
|
required: false
|
|
default: 'false'
|
|
run_with_real_s3:
|
|
description: 'Whether to pass real s3 credentials to the test suite'
|
|
required: false
|
|
default: 'false'
|
|
real_s3_bucket:
|
|
description: 'Bucket name for real s3 tests'
|
|
required: false
|
|
default: ''
|
|
real_s3_region:
|
|
description: 'Region name for real s3 tests'
|
|
required: false
|
|
default: ''
|
|
rerun_failed:
|
|
description: 'Whether to rerun failed tests'
|
|
required: false
|
|
default: 'false'
|
|
pg_version:
|
|
description: 'Postgres version to use for tests'
|
|
required: false
|
|
default: 'v16'
|
|
benchmark_durations:
|
|
description: 'benchmark durations JSON'
|
|
required: false
|
|
default: '{}'
|
|
aws-oicd-role-arn:
|
|
description: 'OIDC role arn to interract with S3'
|
|
required: true
|
|
|
|
runs:
|
|
using: "composite"
|
|
steps:
|
|
- name: Get Neon artifact
|
|
if: inputs.build_type != 'remote'
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build_type }}-artifact
|
|
path: /tmp/neon
|
|
aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}
|
|
|
|
- name: Download Neon binaries for the previous release
|
|
if: inputs.build_type != 'remote'
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build_type }}-artifact
|
|
path: /tmp/neon-previous
|
|
prefix: latest
|
|
aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}
|
|
|
|
- name: Download compatibility snapshot
|
|
if: inputs.build_type != 'remote'
|
|
uses: ./.github/actions/download
|
|
with:
|
|
name: compatibility-snapshot-${{ runner.arch }}-${{ inputs.build_type }}-pg${{ inputs.pg_version }}
|
|
path: /tmp/compatibility_snapshot_pg${{ inputs.pg_version }}
|
|
prefix: latest
|
|
# The lack of compatibility snapshot (for example, for the new Postgres version)
|
|
# shouldn't fail the whole job. Only relevant test should fail.
|
|
skip-if-does-not-exist: true
|
|
aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}
|
|
|
|
- name: Checkout
|
|
if: inputs.needs_postgres_source == 'true'
|
|
uses: actions/checkout@v4
|
|
with:
|
|
submodules: true
|
|
|
|
- name: Cache poetry deps
|
|
uses: actions/cache@v4
|
|
with:
|
|
path: ~/.cache/pypoetry/virtualenvs
|
|
key: v2-${{ runner.os }}-${{ runner.arch }}-python-deps-bookworm-${{ hashFiles('poetry.lock') }}
|
|
|
|
- name: Install Python deps
|
|
shell: bash -euxo pipefail {0}
|
|
run: ./scripts/pysync
|
|
|
|
- name: Run pytest
|
|
env:
|
|
NEON_BIN: /tmp/neon/bin
|
|
COMPATIBILITY_NEON_BIN: /tmp/neon-previous/bin
|
|
COMPATIBILITY_POSTGRES_DISTRIB_DIR: /tmp/neon-previous/pg_install
|
|
TEST_OUTPUT: /tmp/test_output
|
|
BUILD_TYPE: ${{ inputs.build_type }}
|
|
COMPATIBILITY_SNAPSHOT_DIR: /tmp/compatibility_snapshot_pg${{ inputs.pg_version }}
|
|
ALLOW_BACKWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'backward compatibility breakage')
|
|
ALLOW_FORWARD_COMPATIBILITY_BREAKAGE: contains(github.event.pull_request.labels.*.name, 'forward compatibility breakage')
|
|
RERUN_FAILED: ${{ inputs.rerun_failed }}
|
|
PG_VERSION: ${{ inputs.pg_version }}
|
|
shell: bash -euxo pipefail {0}
|
|
run: |
|
|
# PLATFORM will be embedded in the perf test report
|
|
# and it is needed to distinguish different environments
|
|
export PLATFORM=${PLATFORM:-github-actions-selfhosted}
|
|
export POSTGRES_DISTRIB_DIR=${POSTGRES_DISTRIB_DIR:-/tmp/neon/pg_install}
|
|
export DEFAULT_PG_VERSION=${PG_VERSION#v}
|
|
export LD_LIBRARY_PATH=${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/lib
|
|
export BENCHMARK_CONNSTR=${BENCHMARK_CONNSTR:-}
|
|
export ASAN_OPTIONS=detect_leaks=0:detect_stack_use_after_return=0:abort_on_error=1:strict_string_checks=1:check_initialization_order=1:strict_init_order=1
|
|
export UBSAN_OPTIONS=abort_on_error=1:print_stacktrace=1
|
|
|
|
if [ "${BUILD_TYPE}" = "remote" ]; then
|
|
export REMOTE_ENV=1
|
|
fi
|
|
|
|
PERF_REPORT_DIR="$(realpath test_runner/perf-report-local)"
|
|
rm -rf $PERF_REPORT_DIR
|
|
|
|
TEST_SELECTION="test_runner/${{ inputs.test_selection }}"
|
|
EXTRA_PARAMS="${{ inputs.extra_params }}"
|
|
if [ -z "$TEST_SELECTION" ]; then
|
|
echo "test_selection must be set"
|
|
exit 1
|
|
fi
|
|
if [[ "${{ inputs.run_in_parallel }}" == "true" ]]; then
|
|
# -n sets the number of parallel processes that pytest-xdist will run
|
|
EXTRA_PARAMS="-n12 $EXTRA_PARAMS"
|
|
|
|
# --dist=loadgroup points tests marked with @pytest.mark.xdist_group
|
|
# to the same worker to make @pytest.mark.order work with xdist
|
|
EXTRA_PARAMS="--dist=loadgroup $EXTRA_PARAMS"
|
|
fi
|
|
|
|
if [[ "${{ inputs.run_with_real_s3 }}" == "true" ]]; then
|
|
echo "REAL S3 ENABLED"
|
|
export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
|
|
export REMOTE_STORAGE_S3_BUCKET=${{ inputs.real_s3_bucket }}
|
|
export REMOTE_STORAGE_S3_REGION=${{ inputs.real_s3_region }}
|
|
fi
|
|
|
|
if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then
|
|
mkdir -p "$PERF_REPORT_DIR"
|
|
EXTRA_PARAMS="--out-dir $PERF_REPORT_DIR $EXTRA_PARAMS"
|
|
fi
|
|
|
|
if [ "${RERUN_FAILED}" == "true" ]; then
|
|
EXTRA_PARAMS="--reruns 2 $EXTRA_PARAMS"
|
|
fi
|
|
|
|
# We use pytest-split plugin to run benchmarks in parallel on different CI runners
|
|
if [ "${TEST_SELECTION}" = "test_runner/performance" ] && [ "${{ inputs.build_type }}" != "remote" ]; then
|
|
mkdir -p $TEST_OUTPUT
|
|
echo '${{ inputs.benchmark_durations || '{}' }}' > $TEST_OUTPUT/benchmark_durations.json
|
|
|
|
EXTRA_PARAMS="--durations-path $TEST_OUTPUT/benchmark_durations.json $EXTRA_PARAMS"
|
|
fi
|
|
|
|
if [[ $BUILD_TYPE == "debug" && $RUNNER_ARCH == 'X64' ]]; then
|
|
cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
|
|
else
|
|
cov_prefix=()
|
|
fi
|
|
|
|
# Wake up the cluster if we use remote neon instance
|
|
if [ "${{ inputs.build_type }}" = "remote" ] && [ -n "${BENCHMARK_CONNSTR}" ]; then
|
|
QUERIES=("SELECT version()")
|
|
if [[ "${PLATFORM}" = "neon"* ]]; then
|
|
QUERIES+=("SHOW neon.tenant_id")
|
|
QUERIES+=("SHOW neon.timeline_id")
|
|
fi
|
|
|
|
for q in "${QUERIES[@]}"; do
|
|
${POSTGRES_DISTRIB_DIR}/v${DEFAULT_PG_VERSION}/bin/psql ${BENCHMARK_CONNSTR} -c "${q}"
|
|
done
|
|
fi
|
|
|
|
# Run the tests.
|
|
#
|
|
# --alluredir saves test results in Allure format (in a specified directory)
|
|
# --verbose prints name of each test (helpful when there are
|
|
# multiple tests in one file)
|
|
# -rA prints summary in the end
|
|
# -s is not used to prevent pytest from capturing output, because tests are running
|
|
# in parallel and logs are mixed between different tests
|
|
#
|
|
mkdir -p $TEST_OUTPUT/allure/results
|
|
"${cov_prefix[@]}" ./scripts/pytest \
|
|
--alluredir=$TEST_OUTPUT/allure/results \
|
|
--tb=short \
|
|
--verbose \
|
|
-rA $TEST_SELECTION $EXTRA_PARAMS
|
|
|
|
if [[ "${{ inputs.save_perf_report }}" == "true" ]]; then
|
|
export REPORT_FROM="$PERF_REPORT_DIR"
|
|
export REPORT_TO="$PLATFORM"
|
|
scripts/generate_and_push_perf_report.sh
|
|
fi
|
|
|
|
- name: Upload compatibility snapshot
|
|
# Note, that we use `github.base_ref` which is a target branch for a PR
|
|
if: github.event_name == 'pull_request' && github.base_ref == 'release'
|
|
uses: ./.github/actions/upload
|
|
with:
|
|
name: compatibility-snapshot-${{ runner.arch }}-${{ inputs.build_type }}-pg${{ inputs.pg_version }}
|
|
# Directory is created by test_compatibility.py::test_create_snapshot, keep the path in sync with the test
|
|
path: /tmp/test_output/compatibility_snapshot_pg${{ inputs.pg_version }}/
|
|
# The lack of compatibility snapshot shouldn't fail the job
|
|
# (for example if we didn't run the test for non build-and-test workflow)
|
|
skip-if-does-not-exist: true
|
|
aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}
|
|
|
|
- uses: aws-actions/configure-aws-credentials@v4
|
|
if: ${{ !cancelled() }}
|
|
with:
|
|
aws-region: eu-central-1
|
|
role-to-assume: ${{ inputs.aws-oicd-role-arn }}
|
|
role-duration-seconds: 3600 # 1 hour should be more than enough to upload report
|
|
|
|
- name: Upload test results
|
|
if: ${{ !cancelled() }}
|
|
uses: ./.github/actions/allure-report-store
|
|
with:
|
|
report-dir: /tmp/test_output/allure/results
|
|
unique-key: ${{ inputs.build_type }}-${{ inputs.pg_version }}
|
|
aws-oicd-role-arn: ${{ inputs.aws-oicd-role-arn }}
|