From bb06d281ea8cdcde69461e5be31e05efe511e345 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Fri, 12 May 2023 15:28:51 +0100 Subject: [PATCH] Run regressions tests on both Postgres 14 and 15 (#4192) This PR adds tests runs on Postgres 15 and created unified Allure report with results for all tests. - Split `.github/actions/allure-report` into `.github/actions/allure-report-store` and `.github/actions/allure-report-generate` - Add debug or release pytest parameter for all tests (depending on `BUILD_TYPE` env variable) - Add Postgres version as a pytest parameter for all tests (depending on `DEFAULT_PG_VERSION` env variable) - Fix `test_wal_restore` and `restore_from_wal.sh` to support path with `[`/`]` in it (fixed by applying spellcheck to the script and fixing all warnings), `restore_from_wal_archive.sh` is deleted as unused. - All known failures on Postgres 15 marked with xfail --- .../actions/allure-report-generate/action.yml | 184 +++++++++++++ .../actions/allure-report-store/action.yml | 72 +++++ .github/actions/allure-report/action.yml | 254 ------------------ .../actions/run-python-test-set/action.yml | 13 +- .github/workflows/benchmarking.yml | 27 +- .github/workflows/build_and_test.yml | 68 ++--- libs/utils/scripts/restore_from_wal.sh | 28 +- .../utils/scripts/restore_from_wal_archive.sh | 20 -- scripts/pr-comment-test-report.js | 158 +++++++---- test_runner/conftest.py | 1 + test_runner/fixtures/allure.py | 25 ++ test_runner/fixtures/pg_version.py | 14 + test_runner/regress/test_compatibility.py | 27 +- test_runner/regress/test_hot_standby.py | 2 + test_runner/regress/test_pg_regress.py | 2 + test_runner/regress/test_tenant_size.py | 2 + test_runner/regress/test_wal_restore.py | 20 +- 17 files changed, 488 insertions(+), 429 deletions(-) create mode 100644 .github/actions/allure-report-generate/action.yml create mode 100644 .github/actions/allure-report-store/action.yml delete mode 100644 .github/actions/allure-report/action.yml delete mode 100755 libs/utils/scripts/restore_from_wal_archive.sh create mode 100644 test_runner/fixtures/allure.py diff --git a/.github/actions/allure-report-generate/action.yml b/.github/actions/allure-report-generate/action.yml new file mode 100644 index 0000000000..07120c4c8a --- /dev/null +++ b/.github/actions/allure-report-generate/action.yml @@ -0,0 +1,184 @@ +name: 'Create Allure report' +description: 'Generate Allure report from uploaded by actions/allure-report-store tests results' + +outputs: + report-url: + description: 'Allure report URL' + value: ${{ steps.generate-report.outputs.report-url }} + report-json-url: + description: 'Allure report JSON URL' + value: ${{ steps.generate-report.outputs.report-json-url }} + +runs: + using: "composite" + + steps: + # We're using some of env variables quite offen, so let's set them once. + # + # It would be nice to have them set in common runs.env[0] section, but it doesn't work[1] + # + # - [0] https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#runsenv + # - [1] https://github.com/neondatabase/neon/pull/3907#discussion_r1154703456 + # + - name: Set variables + shell: bash -euxo pipefail {0} + run: | + PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH" || true) + if [ "${PR_NUMBER}" != "null" ]; then + BRANCH_OR_PR=pr-${PR_NUMBER} + elif [ "${GITHUB_REF_NAME}" = "main" ] || [ "${GITHUB_REF_NAME}" = "release" ]; then + # Shortcut for special branches + BRANCH_OR_PR=${GITHUB_REF_NAME} + else + BRANCH_OR_PR=branch-$(printf "${GITHUB_REF_NAME}" | tr -c "[:alnum:]._-" "-") + fi + + LOCK_FILE=reports/${BRANCH_OR_PR}/lock.txt + + WORKDIR=/tmp/${BRANCH_OR_PR}-$(date +%s) + mkdir -p ${WORKDIR} + + echo "BRANCH_OR_PR=${BRANCH_OR_PR}" >> $GITHUB_ENV + echo "LOCK_FILE=${LOCK_FILE}" >> $GITHUB_ENV + echo "WORKDIR=${WORKDIR}" >> $GITHUB_ENV + echo "BUCKET=${BUCKET}" >> $GITHUB_ENV + env: + BUCKET: neon-github-public-dev + + # TODO: We can replace with a special docker image with Java and Allure pre-installed + - uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '17' + + - name: Install Allure + shell: bash -euxo pipefail {0} + run: | + if ! which allure; then + ALLURE_ZIP=allure-${ALLURE_VERSION}.zip + wget -q https://github.com/allure-framework/allure2/releases/download/${ALLURE_VERSION}/${ALLURE_ZIP} + echo "${ALLURE_ZIP_MD5} ${ALLURE_ZIP}" | md5sum -c + unzip -q ${ALLURE_ZIP} + echo "$(pwd)/allure-${ALLURE_VERSION}/bin" >> $GITHUB_PATH + rm -f ${ALLURE_ZIP} + fi + env: + ALLURE_VERSION: 2.22.0 + ALLURE_ZIP_MD5: d5c9f0989b896482536956340a7d5ec9 + + # Potentially we could have several running build for the same key (for example, for the main branch), so we use improvised lock for this + - name: Acquire lock + shell: bash -euxo pipefail {0} + run: | + LOCK_TIMEOUT=300 # seconds + + LOCK_CONTENT="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + echo ${LOCK_CONTENT} > ${WORKDIR}/lock.txt + + # Do it up to 5 times to avoid race condition + for _ in $(seq 1 5); do + for i in $(seq 1 ${LOCK_TIMEOUT}); do + LOCK_ACQUIRED=$(aws s3api head-object --bucket neon-github-public-dev --key ${LOCK_FILE} | jq --raw-output '.LastModified' || true) + # `date --date="..."` is supported only by gnu date (i.e. it doesn't work on BSD/macOS) + if [ -z "${LOCK_ACQUIRED}" ] || [ "$(( $(date +%s) - $(date --date="${LOCK_ACQUIRED}" +%s) ))" -gt "${LOCK_TIMEOUT}" ]; then + break + fi + sleep 1 + done + + aws s3 mv --only-show-errors ${WORKDIR}/lock.txt "s3://${BUCKET}/${LOCK_FILE}" + + # Double-check that exactly THIS run has acquired the lock + aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt + if [ "$(cat lock.txt)" = "${LOCK_CONTENT}" ]; then + break + fi + done + + - name: Generate and publish final Allure report + id: generate-report + shell: bash -euxo pipefail {0} + run: | + REPORT_PREFIX=reports/${BRANCH_OR_PR} + RAW_PREFIX=reports-raw/${BRANCH_OR_PR}/${GITHUB_RUN_ID} + + # Get previously uploaded data for this run + ZSTD_NBTHREADS=0 + + S3_FILEPATHS=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${RAW_PREFIX}/ | jq --raw-output '.Contents[].Key') + if [ -z "$S3_FILEPATHS" ]; then + # There's no previously uploaded data for this $GITHUB_RUN_ID + exit 0 + fi + for S3_FILEPATH in ${S3_FILEPATHS}; do + time aws s3 cp --only-show-errors "s3://${BUCKET}/${S3_FILEPATH}" "${WORKDIR}" + + archive=${WORKDIR}/$(basename $S3_FILEPATH) + mkdir -p ${archive%.tar.zst} + time tar -xf ${archive} -C ${archive%.tar.zst} + rm -f ${archive} + done + + # Get history trend + time aws s3 cp --recursive --only-show-errors "s3://${BUCKET}/${REPORT_PREFIX}/latest/history" "${WORKDIR}/latest/history" || true + + # Generate report + time allure generate --clean --output ${WORKDIR}/report ${WORKDIR}/* + + # Replace a logo link with a redirect to the latest version of the report + sed -i 's| ${WORKDIR}/index.html + + + + Redirecting to ${REPORT_URL} + + EOF + time aws s3 cp --only-show-errors ${WORKDIR}/index.html "s3://${BUCKET}/${REPORT_PREFIX}/latest/index.html" + + echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT + echo "report-json-url=${REPORT_URL%/index.html}/data/suites.json" >> $GITHUB_OUTPUT + + - name: Release lock + if: always() + shell: bash -euxo pipefail {0} + run: | + aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt || exit 0 + + if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" ]; then + aws s3 rm "s3://${BUCKET}/${LOCK_FILE}" + fi + + - name: Cleanup + if: always() + shell: bash -euxo pipefail {0} + run: | + if [ -d "${WORKDIR}" ]; then + rm -rf ${WORKDIR} + fi + + - uses: actions/github-script@v6 + if: always() + env: + REPORT_URL: ${{ steps.generate-report.outputs.report-url }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + with: + script: | + const { REPORT_URL, COMMIT_SHA } = process.env + + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, + repo: context.repo.repo, + sha: `${COMMIT_SHA}`, + state: 'success', + target_url: `${REPORT_URL}`, + context: 'Allure report', + }) diff --git a/.github/actions/allure-report-store/action.yml b/.github/actions/allure-report-store/action.yml new file mode 100644 index 0000000000..7ae9937d42 --- /dev/null +++ b/.github/actions/allure-report-store/action.yml @@ -0,0 +1,72 @@ +name: 'Store Allure results' +description: 'Upload test results to be used by actions/allure-report-generate' + +inputs: + report-dir: + description: 'directory with test results generated by tests' + required: true + unique-key: + description: 'string to distinguish different results in the same run' + required: true + +runs: + using: "composite" + + steps: + - name: Set variables + shell: bash -euxo pipefail {0} + run: | + PR_NUMBER=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH" || true) + if [ "${PR_NUMBER}" != "null" ]; then + BRANCH_OR_PR=pr-${PR_NUMBER} + elif [ "${GITHUB_REF_NAME}" = "main" ] || [ "${GITHUB_REF_NAME}" = "release" ]; then + # Shortcut for special branches + BRANCH_OR_PR=${GITHUB_REF_NAME} + else + BRANCH_OR_PR=branch-$(printf "${GITHUB_REF_NAME}" | tr -c "[:alnum:]._-" "-") + fi + + echo "BRANCH_OR_PR=${BRANCH_OR_PR}" >> $GITHUB_ENV + echo "REPORT_DIR=${REPORT_DIR}" >> $GITHUB_ENV + env: + REPORT_DIR: ${{ inputs.report-dir }} + + - name: Upload test results + shell: bash -euxo pipefail {0} + run: | + REPORT_PREFIX=reports/${BRANCH_OR_PR} + RAW_PREFIX=reports-raw/${BRANCH_OR_PR}/${GITHUB_RUN_ID} + + # Add metadata + cat < ${REPORT_DIR}/executor.json + { + "name": "GitHub Actions", + "type": "github", + "url": "https://${BUCKET}.s3.amazonaws.com/${REPORT_PREFIX}/latest/index.html", + "buildOrder": ${GITHUB_RUN_ID}, + "buildName": "GitHub Actions Run #${GITHUB_RUN_NUMBER}/${GITHUB_RUN_ATTEMPT}", + "buildUrl": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}", + "reportUrl": "https://${BUCKET}.s3.amazonaws.com/${REPORT_PREFIX}/${GITHUB_RUN_ID}/index.html", + "reportName": "Allure Report" + } + EOF + + cat < ${REPORT_DIR}/environment.properties + COMMIT_SHA=${COMMIT_SHA} + EOF + + ARCHIVE="${UNIQUE_KEY}-${GITHUB_RUN_ATTEMPT}-$(date +%s).tar.zst" + ZSTD_NBTHREADS=0 + + time tar -C ${REPORT_DIR} -cf ${ARCHIVE} --zstd . + time aws s3 mv --only-show-errors ${ARCHIVE} "s3://${BUCKET}/${RAW_PREFIX}/${ARCHIVE}" + env: + UNIQUE_KEY: ${{ inputs.unique-key }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + BUCKET: neon-github-public-dev + + - name: Cleanup + if: always() + shell: bash -euxo pipefail {0} + run: | + rm -rf ${REPORT_DIR} diff --git a/.github/actions/allure-report/action.yml b/.github/actions/allure-report/action.yml deleted file mode 100644 index 9a1037064a..0000000000 --- a/.github/actions/allure-report/action.yml +++ /dev/null @@ -1,254 +0,0 @@ -name: 'Create Allure report' -description: 'Create and publish Allure report' - -inputs: - action: - desctiption: 'generate or store' - required: true - build_type: - description: '`build_type` from run-python-test-set action' - required: true - test_selection: - description: '`test_selector` from run-python-test-set action' - required: false -outputs: - report-url: - description: 'Allure report URL' - value: ${{ steps.generate-report.outputs.report-url }} - report-json-url: - description: 'Allure report JSON URL' - value: ${{ steps.generate-report.outputs.report-json-url }} - -runs: - using: "composite" - - steps: - # We're using some of env variables quite offen, so let's set them once. - # - # It would be nice to have them set in common runs.env[0] section, but it doesn't work[1] - # - # - [0] https://docs.github.com/en/actions/creating-actions/metadata-syntax-for-github-actions#runsenv - # - [1] https://github.com/neondatabase/neon/pull/3907#discussion_r1154703456 - # - - name: Set common environment variables - shell: bash -euxo pipefail {0} - run: | - echo "BUILD_TYPE=${BUILD_TYPE}" >> $GITHUB_ENV - echo "BUCKET=${BUCKET}" >> $GITHUB_ENV - echo "TEST_OUTPUT=${TEST_OUTPUT}" >> $GITHUB_ENV - env: - BUILD_TYPE: ${{ inputs.build_type }} - BUCKET: neon-github-public-dev - TEST_OUTPUT: /tmp/test_output - - - name: Validate input parameters - shell: bash -euxo pipefail {0} - run: | - if [ "${{ inputs.action }}" != "store" ] && [ "${{ inputs.action }}" != "generate" ]; then - echo >&2 "Unknown inputs.action type '${{ inputs.action }}'; allowed 'generate' or 'store' only" - exit 1 - fi - - if [ -z "${{ inputs.test_selection }}" ] && [ "${{ inputs.action }}" == "store" ]; then - echo >&2 "inputs.test_selection must be set for 'store' action" - exit 2 - fi - - - name: Calculate variables - id: calculate-vars - shell: bash -euxo pipefail {0} - run: | - # TODO: for manually triggered workflows (via workflow_dispatch) we need to have a separate key - - pr_number=$(jq --raw-output .pull_request.number "$GITHUB_EVENT_PATH" || true) - if [ "${pr_number}" != "null" ]; then - key=pr-${pr_number} - elif [ "${GITHUB_REF_NAME}" = "main" ]; then - # Shortcut for a special branch - key=main - elif [ "${GITHUB_REF_NAME}" = "release" ]; then - # Shortcut for a special branch - key=release - else - key=branch-$(printf "${GITHUB_REF_NAME}" | tr -c "[:alnum:]._-" "-") - fi - echo "KEY=${key}" >> $GITHUB_OUTPUT - - # Sanitize test selection to remove `/` and any other special characters - # Use printf instead of echo to avoid having `\n` at the end of the string - test_selection=$(printf "${{ inputs.test_selection }}" | tr -c "[:alnum:]._-" "-" ) - echo "TEST_SELECTION=${test_selection}" >> $GITHUB_OUTPUT - - - uses: actions/setup-java@v3 - if: ${{ inputs.action == 'generate' }} - with: - distribution: 'temurin' - java-version: '17' - - - name: Install Allure - if: ${{ inputs.action == 'generate' }} - shell: bash -euxo pipefail {0} - run: | - if ! which allure; then - ALLURE_ZIP=allure-${ALLURE_VERSION}.zip - wget -q https://github.com/allure-framework/allure2/releases/download/${ALLURE_VERSION}/${ALLURE_ZIP} - echo "${ALLURE_ZIP_MD5} ${ALLURE_ZIP}" | md5sum -c - unzip -q ${ALLURE_ZIP} - echo "$(pwd)/allure-${ALLURE_VERSION}/bin" >> $GITHUB_PATH - rm -f ${ALLURE_ZIP} - fi - env: - ALLURE_VERSION: 2.21.0 - ALLURE_ZIP_MD5: c8db4dd8e2a7882583d569ed2c82879c - - - name: Upload Allure results - if: ${{ inputs.action == 'store' }} - env: - REPORT_PREFIX: reports/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }} - RAW_PREFIX: reports-raw/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }} - TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }} - shell: bash -euxo pipefail {0} - run: | - # Add metadata - cat < $TEST_OUTPUT/allure/results/executor.json - { - "name": "GitHub Actions", - "type": "github", - "url": "https://${BUCKET}.s3.amazonaws.com/${REPORT_PREFIX}/latest/index.html", - "buildOrder": ${GITHUB_RUN_ID}, - "buildName": "GitHub Actions Run #${{ github.run_number }}/${GITHUB_RUN_ATTEMPT}", - "buildUrl": "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}", - "reportUrl": "https://${BUCKET}.s3.amazonaws.com/${REPORT_PREFIX}/${GITHUB_RUN_ID}/index.html", - "reportName": "Allure Report" - } - EOF - cat < $TEST_OUTPUT/allure/results/environment.properties - TEST_SELECTION=${{ inputs.test_selection }} - BUILD_TYPE=${BUILD_TYPE} - EOF - - ARCHIVE="${GITHUB_RUN_ID}-${TEST_SELECTION}-${GITHUB_RUN_ATTEMPT}-$(date +%s).tar.zst" - ZSTD_NBTHREADS=0 - - tar -C ${TEST_OUTPUT}/allure/results -cf ${ARCHIVE} --zstd . - aws s3 mv --only-show-errors ${ARCHIVE} "s3://${BUCKET}/${RAW_PREFIX}/${ARCHIVE}" - - # Potentially we could have several running build for the same key (for example for the main branch), so we use improvised lock for this - - name: Acquire Allure lock - if: ${{ inputs.action == 'generate' }} - shell: bash -euxo pipefail {0} - env: - LOCK_FILE: reports/${{ steps.calculate-vars.outputs.KEY }}/lock.txt - TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }} - run: | - LOCK_TIMEOUT=300 # seconds - - for _ in $(seq 1 5); do - for i in $(seq 1 ${LOCK_TIMEOUT}); do - LOCK_ADDED=$(aws s3api head-object --bucket neon-github-public-dev --key ${LOCK_FILE} | jq --raw-output '.LastModified' || true) - # `date --date="..."` is supported only by gnu date (i.e. it doesn't work on BSD/macOS) - if [ -z "${LOCK_ADDED}" ] || [ "$(( $(date +%s) - $(date --date="${LOCK_ADDED}" +%s) ))" -gt "${LOCK_TIMEOUT}" ]; then - break - fi - sleep 1 - done - echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" > lock.txt - aws s3 mv --only-show-errors lock.txt "s3://${BUCKET}/${LOCK_FILE}" - - # A double-check that exactly WE have acquired the lock - aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt - if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" ]; then - break - fi - done - - - name: Generate and publish final Allure report - if: ${{ inputs.action == 'generate' }} - id: generate-report - env: - REPORT_PREFIX: reports/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }} - RAW_PREFIX: reports-raw/${{ steps.calculate-vars.outputs.KEY }}/${{ inputs.build_type }} - shell: bash -euxo pipefail {0} - run: | - # Get previously uploaded data for this run - ZSTD_NBTHREADS=0 - - s3_filepaths=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${RAW_PREFIX}/${GITHUB_RUN_ID}- | jq --raw-output '.Contents[].Key') - if [ -z "$s3_filepaths" ]; then - # There's no previously uploaded data for this run - exit 0 - fi - for s3_filepath in ${s3_filepaths}; do - aws s3 cp --only-show-errors "s3://${BUCKET}/${s3_filepath}" "${TEST_OUTPUT}/allure/" - - archive=${TEST_OUTPUT}/allure/$(basename $s3_filepath) - mkdir -p ${archive%.tar.zst} - tar -xf ${archive} -C ${archive%.tar.zst} - rm -f ${archive} - done - - # Get history trend - aws s3 cp --recursive --only-show-errors "s3://${BUCKET}/${REPORT_PREFIX}/latest/history" "${TEST_OUTPUT}/allure/latest/history" || true - - # Generate report - allure generate --clean --output $TEST_OUTPUT/allure/report $TEST_OUTPUT/allure/* - - # Replace a logo link with a redirect to the latest version of the report - sed -i 's| ${TEST_OUTPUT}/allure/index.html - - - - Redirecting to ${REPORT_URL} - - EOF - aws s3 cp --only-show-errors ${TEST_OUTPUT}/allure/index.html "s3://${BUCKET}/${REPORT_PREFIX}/latest/index.html" - - echo "[Allure Report](${REPORT_URL})" >> ${GITHUB_STEP_SUMMARY} - echo "report-url=${REPORT_URL}" >> $GITHUB_OUTPUT - echo "report-json-url=${REPORT_URL%/index.html}/data/suites.json" >> $GITHUB_OUTPUT - - - name: Release Allure lock - if: ${{ inputs.action == 'generate' && always() }} - shell: bash -euxo pipefail {0} - env: - LOCK_FILE: reports/${{ steps.calculate-vars.outputs.KEY }}/lock.txt - TEST_SELECTION: ${{ steps.calculate-vars.outputs.TEST_SELECTION }} - run: | - aws s3 cp --only-show-errors "s3://${BUCKET}/${LOCK_FILE}" ./lock.txt || exit 0 - - if [ "$(cat lock.txt)" = "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${TEST_SELECTION}" ]; then - aws s3 rm "s3://${BUCKET}/${LOCK_FILE}" - fi - - - name: Cleanup - if: always() - shell: bash -euxo pipefail {0} - run: | - rm -rf ${TEST_OUTPUT}/allure - - - uses: actions/github-script@v6 - if: ${{ inputs.action == 'generate' && always() }} - env: - REPORT_URL: ${{ steps.generate-report.outputs.report-url }} - SHA: ${{ github.event.pull_request.head.sha || github.sha }} - with: - script: | - const { REPORT_URL, BUILD_TYPE, SHA } = process.env - - await github.rest.repos.createCommitStatus({ - owner: context.repo.owner, - repo: context.repo.repo, - sha: `${SHA}`, - state: 'success', - target_url: `${REPORT_URL}`, - context: `Allure report / ${BUILD_TYPE}`, - }) diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 115f555913..1dd294d17a 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -197,14 +197,13 @@ runs: uses: ./.github/actions/upload with: name: compatibility-snapshot-${{ inputs.build_type }}-pg14-${{ github.run_id }} - # The path includes a test name (test_create_snapshot) and directory that the test creates (compatibility_snapshot_pg14), keep the path in sync with the test - path: /tmp/test_output/test_create_snapshot/compatibility_snapshot_pg14/ + # Directory is created by test_compatibility.py::test_create_snapshot, keep the path in sync with the test + path: /tmp/test_output/compatibility_snapshot_pg14/ prefix: latest - - name: Create Allure report + - name: Upload test results if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report + uses: ./.github/actions/allure-report-store with: - action: store - build_type: ${{ inputs.build_type }} - test_selection: ${{ inputs.test_selection }} + report-dir: /tmp/test_output/allure/results + unique-key: ${{ inputs.test_selection }}-${{ inputs.build_type }} diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index a5a27e59a8..11363b2407 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -93,10 +93,7 @@ jobs: - name: Create Allure report if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report - with: - action: generate - build_type: ${{ env.BUILD_TYPE }} + uses: ./.github/actions/allure-report-generate - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} @@ -283,10 +280,7 @@ jobs: - name: Create Allure report if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report - with: - action: generate - build_type: ${{ env.BUILD_TYPE }} + uses: ./.github/actions/allure-report-generate - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} @@ -380,10 +374,7 @@ jobs: - name: Create Allure report if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report - with: - action: generate - build_type: ${{ env.BUILD_TYPE }} + uses: ./.github/actions/allure-report-generate - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} @@ -476,10 +467,7 @@ jobs: - name: Create Allure report if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report - with: - action: generate - build_type: ${{ env.BUILD_TYPE }} + uses: ./.github/actions/allure-report-generate - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} @@ -566,16 +554,13 @@ jobs: - name: Create Allure report if: ${{ !cancelled() }} - uses: ./.github/actions/allure-report - with: - action: generate - build_type: ${{ env.BUILD_TYPE }} + uses: ./.github/actions/allure-report-generate - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} uses: slackapi/slack-github-action@v1 with: channel-id: "C033QLM5P7D" # dev-staging-stream - slack-message: "Periodic TPC-H perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + slack-message: "Periodic User example perf testing ${{ matrix.platform }}: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index fd150f642b..5a09f0b4aa 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -330,6 +330,7 @@ jobs: fail-fast: false matrix: build_type: [ debug, release ] + pg_version: [ v14, v15 ] steps: - name: Checkout uses: actions/checkout@v3 @@ -350,11 +351,12 @@ jobs: real_s3_secret_access_key: "${{ secrets.AWS_SECRET_ACCESS_KEY_CI_TESTS_S3 }}" rerun_flaky: true env: + DEFAULT_PG_VERSION: ${{ matrix.pg_version }} TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR }} CHECK_ONDISK_DATA_COMPATIBILITY: nonempty - name: Merge and upload coverage data - if: matrix.build_type == 'debug' + if: matrix.build_type == 'debug' && matrix.pg_version == 'v14' uses: ./.github/actions/save-coverage-data benchmarks: @@ -399,21 +401,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Create Allure report (debug) + - name: Create Allure report if: ${{ !cancelled() }} - id: create-allure-report-debug - uses: ./.github/actions/allure-report - with: - action: generate - build_type: debug - - - name: Create Allure report (release) - if: ${{ !cancelled() }} - id: create-allure-report-release - uses: ./.github/actions/allure-report - with: - action: generate - build_type: release + id: create-allure-report + uses: ./.github/actions/allure-report-generate - uses: actions/github-script@v6 if: > @@ -423,52 +414,37 @@ jobs: # Retry script for 5XX server errors: https://github.com/actions/github-script#retries retries: 5 script: | - const reports = [{ - buildType: "debug", - reportUrl: "${{ steps.create-allure-report-debug.outputs.report-url }}", - jsonUrl: "${{ steps.create-allure-report-debug.outputs.report-json-url }}", - }, { - buildType: "release", - reportUrl: "${{ steps.create-allure-report-release.outputs.report-url }}", - jsonUrl: "${{ steps.create-allure-report-release.outputs.report-json-url }}", - }] + const report = { + reportUrl: "${{ steps.create-allure-report.outputs.report-url }}", + reportJsonUrl: "${{ steps.create-allure-report.outputs.report-json-url }}", + } const script = require("./scripts/pr-comment-test-report.js") await script({ github, context, fetch, - reports, + report, }) - name: Store Allure test stat in the DB - if: > - !cancelled() && ( - steps.create-allure-report-debug.outputs.report-url || - steps.create-allure-report-release.outputs.report-url - ) + if: ${{ !cancelled() && steps.create-allure-report.outputs.report-json-url }} env: - SHA: ${{ github.event.pull_request.head.sha || github.sha }} - REPORT_JSON_URL_DEBUG: ${{ steps.create-allure-report-debug.outputs.report-json-url }} - REPORT_JSON_URL_RELEASE: ${{ steps.create-allure-report-release.outputs.report-json-url }} + COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + REPORT_JSON_URL: ${{ steps.create-allure-report.outputs.report-json-url }} TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR }} run: | ./scripts/pysync - for report_url in $REPORT_JSON_URL_DEBUG $REPORT_JSON_URL_RELEASE; do - if [ -z "$report_url" ]; then - continue - fi + curl --fail --output suites.json "${REPORT_JSON_URL}" + export BUILD_TYPE=unified + export DATABASE_URL="$TEST_RESULT_CONNSTR" - if [[ "$report_url" == "$REPORT_JSON_URL_DEBUG" ]]; then - BUILD_TYPE=debug - else - BUILD_TYPE=release - fi - - curl --fail --output suites.json "${report_url}" - DATABASE_URL="$TEST_RESULT_CONNSTR" poetry run python3 scripts/ingest_regress_test_result.py --revision ${SHA} --reference ${GITHUB_REF} --build-type ${BUILD_TYPE} --ingest suites.json - done + poetry run python3 scripts/ingest_regress_test_result.py \ + --revision ${COMMIT_SHA} \ + --reference ${GITHUB_REF} \ + --build-type ${BUILD_TYPE} \ + --ingest suites.json coverage-report: runs-on: [ self-hosted, gen3, small ] diff --git a/libs/utils/scripts/restore_from_wal.sh b/libs/utils/scripts/restore_from_wal.sh index 9bd860affb..92cd164b7d 100755 --- a/libs/utils/scripts/restore_from_wal.sh +++ b/libs/utils/scripts/restore_from_wal.sh @@ -1,21 +1,21 @@ #!/bin/bash + +set -euxo pipefail + PG_BIN=$1 WAL_PATH=$2 DATA_DIR=$3 PORT=$4 -SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-` -rm -fr $DATA_DIR -env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U cloud_admin -D $DATA_DIR --sysid=$SYSID -echo port=$PORT >> $DATA_DIR/postgresql.conf -REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-` +SYSID=$(od -A n -j 24 -N 8 -t d8 "$WAL_PATH"/000000010000000000000002* | cut -c 3-) +rm -fr "$DATA_DIR" +env -i LD_LIBRARY_PATH="$PG_BIN"/../lib "$PG_BIN"/initdb -E utf8 -U cloud_admin -D "$DATA_DIR" --sysid="$SYSID" +echo port="$PORT" >> "$DATA_DIR"/postgresql.conf +REDO_POS=0x$("$PG_BIN"/pg_controldata -D "$DATA_DIR" | grep -F "REDO location"| cut -c 42-) declare -i WAL_SIZE=$REDO_POS+114 -$PG_BIN/pg_ctl -D $DATA_DIR -l logfile start -$PG_BIN/pg_ctl -D $DATA_DIR -l logfile stop -m immediate -cp $DATA_DIR/pg_wal/000000010000000000000001 . -cp $WAL_PATH/* $DATA_DIR/pg_wal/ -if [ -f $DATA_DIR/pg_wal/*.partial ] -then - (cd $DATA_DIR/pg_wal ; for partial in \*.partial ; do mv $partial `basename $partial .partial` ; done) -fi -dd if=000000010000000000000001 of=$DATA_DIR/pg_wal/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc +"$PG_BIN"/pg_ctl -D "$DATA_DIR" -l logfile start +"$PG_BIN"/pg_ctl -D "$DATA_DIR" -l logfile stop -m immediate +cp "$DATA_DIR"/pg_wal/000000010000000000000001 . +cp "$WAL_PATH"/* "$DATA_DIR"/pg_wal/ +for partial in "$DATA_DIR"/pg_wal/*.partial ; do mv "$partial" "${partial%.partial}" ; done +dd if=000000010000000000000001 of="$DATA_DIR"/pg_wal/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc rm -f 000000010000000000000001 diff --git a/libs/utils/scripts/restore_from_wal_archive.sh b/libs/utils/scripts/restore_from_wal_archive.sh deleted file mode 100755 index ce58b349fc..0000000000 --- a/libs/utils/scripts/restore_from_wal_archive.sh +++ /dev/null @@ -1,20 +0,0 @@ -PG_BIN=$1 -WAL_PATH=$2 -DATA_DIR=$3 -PORT=$4 -SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-` -rm -fr $DATA_DIR /tmp/pg_wals -mkdir /tmp/pg_wals -env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U cloud_admin -D $DATA_DIR --sysid=$SYSID -echo port=$PORT >> $DATA_DIR/postgresql.conf -REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-` -declare -i WAL_SIZE=$REDO_POS+114 -cp $WAL_PATH/* /tmp/pg_wals -if [ -f $DATA_DIR/pg_wal/*.partial ] -then - (cd /tmp/pg_wals ; for partial in \*.partial ; do mv $partial `basename $partial .partial` ; done) -fi -dd if=$DATA_DIR/pg_wal/000000010000000000000001 of=/tmp/pg_wals/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc -echo > $DATA_DIR/recovery.signal -rm -f $DATA_DIR/pg_wal/* -echo "restore_command = 'cp /tmp/pg_wals/%f %p'" >> $DATA_DIR/postgresql.conf diff --git a/scripts/pr-comment-test-report.js b/scripts/pr-comment-test-report.js index 134df432ae..7cb2a5494f 100644 --- a/scripts/pr-comment-test-report.js +++ b/scripts/pr-comment-test-report.js @@ -1,6 +1,5 @@ // // The script parses Allure reports and posts a comment with a summary of the test results to the PR. -// It accepts an array of items and creates a comment with a summary for each one (for "release" and "debug", together or separately if any of them failed to be generated). // // The comment is updated on each run with the latest results. // @@ -13,19 +12,37 @@ // github, // context, // fetch, -// reports: [{...}, ...], // each report is expected to have "buildType", "reportUrl", and "jsonUrl" properties +// report: { +// reportUrl: "...", +// reportJsonUrl: "...", +// }, // }) // -module.exports = async ({ github, context, fetch, reports }) => { +// Analog of Python's defaultdict. +// +// const dm = new DefaultMap(() => new DefaultMap(() => [])) +// dm["firstKey"]["secondKey"].push("value") +// +class DefaultMap extends Map { + constructor(getDefaultValue) { + return new Proxy({}, { + get: (target, name) => name in target ? target[name] : (target[name] = getDefaultValue(name)) + }) + } +} + +module.exports = async ({ github, context, fetch, report }) => { // Marker to find the comment in the subsequent runs const startMarker = `` + // Let users know that the comment is updated automatically + const autoupdateNotice = `
The comment gets automatically updated with the latest test results :recycle:
` // GitHub bot id taken from (https://api.github.com/users/github-actions[bot]) const githubActionsBotId = 41898282 // The latest commit in the PR URL const commitUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/pull/${context.payload.number}/commits/${context.payload.pull_request.head.sha}` // Commend body itself - let commentBody = `${startMarker}\n### Test results for ${commitUrl}:\n___\n` + let commentBody = `${startMarker}\n` // Common parameters for GitHub API requests const ownerRepoParams = { @@ -33,76 +50,109 @@ module.exports = async ({ github, context, fetch, reports }) => { repo: context.repo.repo, } - for (const report of reports) { - const {buildType, reportUrl, jsonUrl} = report + const {reportUrl, reportJsonUrl} = report - if (!reportUrl || !jsonUrl) { - commentBody += `#### ${buildType} build: no tests were run or test report is not available\n` - continue - } + if (!reportUrl || !reportJsonUrl) { + commentBody += `#### No tests were run or test report is not available\n` + commentBody += autoupdateNotice + return + } - const suites = await (await fetch(jsonUrl)).json() + const suites = await (await fetch(reportJsonUrl)).json() - // Allure distinguishes "failed" (with an assertion error) and "broken" (with any other error) tests. - // For this report it's ok to treat them in the same way (as failed). - failedTests = [] - passedTests = [] - skippedTests = [] + // Allure distinguishes "failed" (with an assertion error) and "broken" (with any other error) tests. + // For this report it's ok to treat them in the same way (as failed). + const failedTests = new DefaultMap(() => new DefaultMap(() => [])) + const passedTests = new DefaultMap(() => new DefaultMap(() => [])) + const skippedTests = new DefaultMap(() => new DefaultMap(() => [])) + const retriedTests = new DefaultMap(() => new DefaultMap(() => [])) + const flakyTests = new DefaultMap(() => new DefaultMap(() => [])) - retriedTests = [] - retriedStatusChangedTests = [] + let failedTestsCount = 0 + let passedTestsCount = 0 + let skippedTestsCount = 0 + let flakyTestsCount = 0 - for (const parentSuite of suites.children) { - for (const suite of parentSuite.children) { - for (const test of suite.children) { - pytestName = `${parentSuite.name.replace(".", "/")}/${suite.name}.py::${test.name}` - test.pytestName = pytestName + const pgVersions = new Set() + const buildTypes = new Set() - if (test.status === "passed") { - passedTests.push(test); - } else if (test.status === "failed" || test.status === "broken") { - failedTests.push(test); - } else if (test.status === "skipped") { - skippedTests.push(test); - } + for (const parentSuite of suites.children) { + for (const suite of parentSuite.children) { + for (const test of suite.children) { + const {groups: {buildType, pgVersion}} = test.name.match(/[\[-](?debug|release)-pg(?\d+)[-\]]/) - if (test.retriesCount > 0) { - retriedTests.push(test); + pgVersions.add(pgVersion) + buildTypes.add(buildType) - if (test.retriesStatusChange) { - retriedStatusChangedTests.push(test); - } + // Removing build type and PostgreSQL version from the test name to make it shorter + const testName = test.name.replace(new RegExp(`${buildType}-pg${pgVersion}-?`), "").replace("[]", "") + test.pytestName = `${parentSuite.name.replace(".", "/")}/${suite.name}.py::${testName}` + + if (test.status === "passed") { + passedTests[pgVersion][buildType].push(test) + passedTestsCount += 1 + } else if (test.status === "failed" || test.status === "broken") { + failedTests[pgVersion][buildType].push(test) + failedTestsCount += 1 + } else if (test.status === "skipped") { + skippedTests[pgVersion][buildType].push(test) + skippedTestsCount += 1 + } + + if (test.retriesCount > 0) { + retriedTests[pgVersion][buildType].push(test) + + if (test.retriesStatusChange) { + flakyTests[pgVersion][buildType].push(test) + flakyTestsCount += 1 } } } } + } - const totalTestsCount = failedTests.length + passedTests.length + skippedTests.length - commentBody += `#### ${buildType} build: ${totalTestsCount} tests run: ${passedTests.length} passed, ${failedTests.length} failed, ${skippedTests.length} skipped ([full report](${reportUrl}))\n` - if (failedTests.length > 0) { - commentBody += `Failed tests:\n` - for (const test of failedTests) { - const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}` + const totalTestsCount = failedTestsCount + passedTestsCount + skippedTestsCount + commentBody += `### ${totalTestsCount} tests run: ${passedTestsCount} passed, ${failedTestsCount} failed, ${skippedTestsCount} skipped ([full report](${reportUrl}) for ${commitUrl})\n___\n` - commentBody += `- [\`${test.pytestName}\`](${allureLink})` - if (test.retriesCount > 0) { - commentBody += ` (ran [${test.retriesCount + 1} times](${allureLink}/retries))` + // Print test resuls from the newest to the oldest PostgreSQL version for release and debug builds. + for (const pgVersion of Array.from(pgVersions).sort().reverse()) { + for (const buildType of Array.from(buildTypes).sort().reverse()) { + if (failedTests[pgVersion][buildType].length > 0) { + commentBody += `#### PostgreSQL ${pgVersion} (${buildType} build)\n\n` + commentBody += `Failed tests:\n` + for (const test of failedTests[pgVersion][buildType]) { + const allureLink = `${reportUrl}#suites/${test.parentUid}/${test.uid}` + + commentBody += `- [\`${test.pytestName}\`](${allureLink})` + if (test.retriesCount > 0) { + commentBody += ` (ran [${test.retriesCount + 1} times](${allureLink}/retries))` + } + commentBody += "\n" } commentBody += "\n" } - commentBody += "\n" } - if (retriedStatusChangedTests > 0) { - commentBody += `Flaky tests:\n` - for (const test of retriedStatusChangedTests) { - const status = test.status === "passed" ? ":white_check_mark:" : ":x:" - commentBody += `- ${status} [\`${test.pytestName}\`](${reportUrl}#suites/${test.parentUid}/${test.uid}/retries)\n` - } - commentBody += "\n" - } - commentBody += "___\n" } + if (flakyTestsCount > 0) { + commentBody += "
\nFlaky tests\n\n" + for (const pgVersion of Array.from(pgVersions).sort().reverse()) { + for (const buildType of Array.from(buildTypes).sort().reverse()) { + if (flakyTests[pgVersion][buildType].length > 0) { + commentBody += `#### PostgreSQL ${pgVersion} (${buildType} build)\n\n` + for (const test of flakyTests[pgVersion][buildType]) { + const status = test.status === "passed" ? ":white_check_mark:" : ":x:" + commentBody += `- ${status} [\`${test.pytestName}\`](${reportUrl}#suites/${test.parentUid}/${test.uid}/retries)\n` + } + commentBody += "\n" + } + } + } + commentBody += "\n
\n" + } + + commentBody += autoupdateNotice + const { data: comments } = await github.rest.issues.listComments({ issue_number: context.payload.number, ...ownerRepoParams, diff --git a/test_runner/conftest.py b/test_runner/conftest.py index 4640861936..4e649e111a 100644 --- a/test_runner/conftest.py +++ b/test_runner/conftest.py @@ -1,5 +1,6 @@ pytest_plugins = ( "fixtures.pg_version", + "fixtures.allure", "fixtures.neon_fixtures", "fixtures.benchmark_fixture", "fixtures.pg_stats", diff --git a/test_runner/fixtures/allure.py b/test_runner/fixtures/allure.py new file mode 100644 index 0000000000..6f40bd2aa2 --- /dev/null +++ b/test_runner/fixtures/allure.py @@ -0,0 +1,25 @@ +import os + +import pytest + +from fixtures.pg_version import DEFAULT_VERSION, PgVersion + +""" +Set of utilities to make Allure report more informative. + +- It adds BUILD_TYPE and DEFAULT_PG_VERSION to the test names (only in test_runner/regress) +to make tests distinguishable in Allure report. +""" + + +@pytest.fixture(scope="function", autouse=True) +def allure_noop(): + pass + + +def pytest_generate_tests(metafunc): + if "test_runner/regress" in metafunc.definition._nodeid: + build_type = os.environ.get("BUILD_TYPE", "DEBUG").lower() + pg_version = PgVersion(os.environ.get("DEFAULT_PG_VERSION", DEFAULT_VERSION)) + + metafunc.parametrize("allure_noop", [f"{build_type}-pg{pg_version}"]) diff --git a/test_runner/fixtures/pg_version.py b/test_runner/fixtures/pg_version.py index 904a274197..554f841d14 100644 --- a/test_runner/fixtures/pg_version.py +++ b/test_runner/fixtures/pg_version.py @@ -46,6 +46,20 @@ class PgVersion(str, enum.Enum): DEFAULT_VERSION: PgVersion = PgVersion.V14 +def skip_on_postgres(version: PgVersion, reason: str): + return pytest.mark.skipif( + PgVersion(os.environ.get("DEFAULT_PG_VERSION", DEFAULT_VERSION)) is version, + reason=reason, + ) + + +def xfail_on_postgres(version: PgVersion, reason: str): + return pytest.mark.xfail( + PgVersion(os.environ.get("DEFAULT_PG_VERSION", DEFAULT_VERSION)) is version, + reason=reason, + ) + + def pytest_addoption(parser: Parser): parser.addoption( "--pg-version", diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index ff61af8fd3..7bc12847b7 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -16,7 +16,7 @@ from fixtures.neon_fixtures import ( ) from fixtures.pageserver.http import PageserverHttpClient from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload -from fixtures.pg_version import PgVersion +from fixtures.pg_version import PgVersion, skip_on_postgres from fixtures.types import Lsn from pytest import FixtureRequest @@ -41,12 +41,15 @@ check_ondisk_data_compatibility_if_enabled = pytest.mark.skipif( ) -# Note: if renaming this test, don't forget to update a reference to it in a workflow file: -# "Upload compatibility snapshot" step in .github/actions/run-python-test-set/action.yml -@check_ondisk_data_compatibility_if_enabled +@skip_on_postgres(PgVersion.V15, "Compatibility tests doesn't support Postgres 15 yet") @pytest.mark.xdist_group("compatibility") @pytest.mark.order(before="test_forward_compatibility") -def test_create_snapshot(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, test_output_dir: Path): +def test_create_snapshot( + neon_env_builder: NeonEnvBuilder, + pg_bin: PgBin, + top_output_dir: Path, + test_output_dir: Path, +): # The test doesn't really test anything # it creates a new snapshot for releases after we tested the current version against the previous snapshot in `test_backward_compatibility`. # @@ -86,10 +89,14 @@ def test_create_snapshot(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, test_o sk.stop() env.pageserver.stop() - shutil.copytree(test_output_dir, test_output_dir / "compatibility_snapshot_pg14") - # Directory `test_output_dir / "compatibility_snapshot_pg14"` is uploaded to S3 in a workflow, keep the name in sync with it + # Directory `compatibility_snapshot_dir` is uploaded to S3 in a workflow, keep the name in sync with it + compatibility_snapshot_dir = top_output_dir / "compatibility_snapshot_pg14" + if compatibility_snapshot_dir.exists(): + shutil.rmtree(compatibility_snapshot_dir) + shutil.copytree(test_output_dir, compatibility_snapshot_dir) +@skip_on_postgres(PgVersion.V15, "Compatibility tests doesn't support Postgres 15 yet") @check_ondisk_data_compatibility_if_enabled @pytest.mark.xdist_group("compatibility") @pytest.mark.order(after="test_create_snapshot") @@ -148,11 +155,13 @@ def test_backward_compatibility( ), "Breaking changes are allowed by ALLOW_BACKWARD_COMPATIBILITY_BREAKAGE, but the test has passed without any breakage" +@skip_on_postgres(PgVersion.V15, "Compatibility tests doesn't support Postgres 15 yet") @check_ondisk_data_compatibility_if_enabled @pytest.mark.xdist_group("compatibility") @pytest.mark.order(after="test_create_snapshot") def test_forward_compatibility( test_output_dir: Path, + top_output_dir: Path, port_distributor: PortDistributor, pg_version: PgVersion, request: FixtureRequest, @@ -174,9 +183,7 @@ def test_forward_compatibility( ), "COMPATIBILITY_POSTGRES_DISTRIB_DIR is not set. It should be set to a pg_install directrory (ideally generated by the previous version of Neon)" compatibility_postgres_distrib_dir = Path(compatibility_postgres_distrib_dir_env).resolve() - compatibility_snapshot_dir = ( - test_output_dir.parent / "test_create_snapshot" / "compatibility_snapshot_pg14" - ) + compatibility_snapshot_dir = top_output_dir / "compatibility_snapshot_pg14" breaking_changes_allowed = ( os.environ.get("ALLOW_FORWARD_COMPATIBILITY_BREAKAGE", "false").lower() == "true" diff --git a/test_runner/regress/test_hot_standby.py b/test_runner/regress/test_hot_standby.py index 12e034cea2..582ac1b17e 100644 --- a/test_runner/regress/test_hot_standby.py +++ b/test_runner/regress/test_hot_standby.py @@ -1,7 +1,9 @@ import pytest from fixtures.neon_fixtures import NeonEnv +from fixtures.pg_version import PgVersion, xfail_on_postgres +@xfail_on_postgres(PgVersion.V15, reason="https://github.com/neondatabase/neon/pull/4182") @pytest.mark.timeout(1800) def test_hot_standby(neon_simple_env: NeonEnv): env = neon_simple_env diff --git a/test_runner/regress/test_pg_regress.py b/test_runner/regress/test_pg_regress.py index 64625ea4ee..a1d2a56d8a 100644 --- a/test_runner/regress/test_pg_regress.py +++ b/test_runner/regress/test_pg_regress.py @@ -5,6 +5,7 @@ from pathlib import Path import pytest from fixtures.neon_fixtures import NeonEnv, check_restored_datadir_content +from fixtures.pg_version import PgVersion, xfail_on_postgres # Run the main PostgreSQL regression tests, in src/test/regress. @@ -71,6 +72,7 @@ def test_pg_regress( # # This runs for a long time, especially in debug mode, so use a larger-than-default # timeout. +@xfail_on_postgres(PgVersion.V15, reason="https://github.com/neondatabase/neon/pull/4213") @pytest.mark.timeout(1800) def test_isolation( neon_simple_env: NeonEnv, diff --git a/test_runner/regress/test_tenant_size.py b/test_runner/regress/test_tenant_size.py index 2d905910f8..60ab268882 100644 --- a/test_runner/regress/test_tenant_size.py +++ b/test_runner/regress/test_tenant_size.py @@ -11,6 +11,7 @@ from fixtures.neon_fixtures import ( wait_for_wal_insert_lsn, ) from fixtures.pageserver.http import PageserverHttpClient +from fixtures.pg_version import PgVersion, xfail_on_postgres from fixtures.types import Lsn, TenantId, TimelineId @@ -512,6 +513,7 @@ def test_single_branch_get_tenant_size_grows( assert size_after == prev, "size after restarting pageserver should not have changed" +@xfail_on_postgres(PgVersion.V15, reason="Test significantly more flaky on Postgres 15") def test_get_tenant_size_with_multiple_branches( neon_env_builder: NeonEnvBuilder, test_output_dir: Path ): diff --git a/test_runner/regress/test_wal_restore.py b/test_runner/regress/test_wal_restore.py index dd944af7eb..f3d3a84c20 100644 --- a/test_runner/regress/test_wal_restore.py +++ b/test_runner/regress/test_wal_restore.py @@ -1,14 +1,20 @@ +import sys from pathlib import Path +import pytest from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, PortDistributor, VanillaPostgres, ) -from fixtures.types import TenantId +from fixtures.types import TenantId, TimelineId +@pytest.mark.skipif( + sys.platform != "linux", + reason="restore_from_wal.sh supports only Linux", +) def test_wal_restore( neon_env_builder: NeonEnvBuilder, pg_bin: PgBin, @@ -22,6 +28,7 @@ def test_wal_restore( endpoint = env.endpoints.create_start("test_wal_restore") endpoint.safe_psql("create table t as select generate_series(1,300000)") tenant_id = TenantId(endpoint.safe_psql("show neon.tenant_id")[0][0]) + timeline_id = TimelineId(endpoint.safe_psql("show neon.timeline_id")[0][0]) env.neon_cli.pageserver_stop() port = port_distributor.get_port() data_dir = test_output_dir / "pgsql.restored" @@ -30,9 +37,16 @@ def test_wal_restore( ) as restored: pg_bin.run_capture( [ - str(base_dir / "libs/utils/scripts/restore_from_wal.sh"), + str(base_dir / "libs" / "utils" / "scripts" / "restore_from_wal.sh"), str(pg_distrib_dir / f"v{env.pg_version}/bin"), - str(test_output_dir / "repo" / "safekeepers" / "sk1" / str(tenant_id) / "*"), + str( + test_output_dir + / "repo" + / "safekeepers" + / "sk1" + / str(tenant_id) + / str(timeline_id) + ), str(data_dir), str(port), ]