diff --git a/.github/actions/download/action.yml b/.github/actions/download/action.yml new file mode 100644 index 0000000000..5aa45164e7 --- /dev/null +++ b/.github/actions/download/action.yml @@ -0,0 +1,56 @@ +name: "Download an artifact" +description: "Custom download action" +inputs: + name: + description: "Artifact name" + required: true + path: + description: "A directory to put artifact into" + default: "." + required: false + skip-if-does-not-exist: + description: "Allow to skip if file doesn't exist, fail otherwise" + default: false + required: false + +runs: + using: "composite" + steps: + - name: Download artifact + id: download-artifact + shell: bash -euxo pipefail {0} + env: + TARGET: ${{ inputs.path }} + ARCHIVE: /tmp/downloads/${{ inputs.name }}.tar.zst + SKIP_IF_DOES_NOT_EXIST: ${{ inputs.skip-if-does-not-exist }} + run: | + BUCKET=neon-github-public-dev + PREFIX=artifacts/${GITHUB_RUN_ID} + FILENAME=$(basename $ARCHIVE) + + S3_KEY=$(aws s3api list-objects-v2 --bucket ${BUCKET} --prefix ${PREFIX} | jq -r '.Contents[].Key' | grep ${FILENAME} | sort --version-sort | tail -1 || true) + if [ -z "${S3_KEY}" ]; then + if [ "${SKIP_IF_DOES_NOT_EXIST}" = "true" ]; then + echo '::set-output name=SKIPPED::true' + exit 0 + else + echo 2>&1 "Neither s3://${BUCKET}/${PREFIX}/${GITHUB_RUN_ATTEMPT}/${FILENAME} nor its version from previous attempts exist" + exit 1 + fi + fi + + echo '::set-output name=SKIPPED::false' + + mkdir -p $(dirname $ARCHIVE) + time aws s3 cp --only-show-errors s3://${BUCKET}/${S3_KEY} ${ARCHIVE} + + - name: Extract artifact + if: ${{ steps.download-artifact.outputs.SKIPPED == 'false' }} + shell: bash -euxo pipefail {0} + env: + TARGET: ${{ inputs.path }} + ARCHIVE: /tmp/downloads/${{ inputs.name }}.tar.zst + run: | + mkdir -p ${TARGET} + time tar -xf ${ARCHIVE} -C ${TARGET} + rm -f ${ARCHIVE} diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 6dc377a809..c9987053ce 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -31,18 +31,11 @@ inputs: runs: using: "composite" steps: - - name: Get Neon artifact for restoration - uses: actions/download-artifact@v3 + - name: Get Neon artifact + uses: ./.github/actions/download with: name: neon-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.rust_toolchain }}-artifact - path: ./neon-artifact/ - - - name: Extract Neon artifact - shell: bash -euxo pipefail {0} - run: | - mkdir -p /tmp/neon/ - tar -xf ./neon-artifact/neon.tar.zst -C /tmp/neon/ - rm -rf ./neon-artifact/ + path: /tmp/neon - name: Checkout if: inputs.needs_postgres_source == 'true' @@ -132,9 +125,7 @@ runs: - name: Upload python test logs if: always() - uses: actions/upload-artifact@v3 + uses: ./.github/actions/upload with: - retention-days: 7 - if-no-files-found: error name: python-test-${{ inputs.test_selection }}-${{ runner.os }}-${{ inputs.build_type }}-${{ inputs.rust_toolchain }}-logs path: /tmp/test_output/ diff --git a/.github/actions/save-coverage-data/action.yml b/.github/actions/save-coverage-data/action.yml index bcfd7cb47e..6fbe19a96e 100644 --- a/.github/actions/save-coverage-data/action.yml +++ b/.github/actions/save-coverage-data/action.yml @@ -8,10 +8,15 @@ runs: shell: bash -euxo pipefail {0} run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge - - name: Upload coverage data - uses: actions/upload-artifact@v3 + - name: Download previous coverage data into the same directory + uses: ./.github/actions/download with: - retention-days: 7 - if-no-files-found: error name: coverage-data-artifact - path: /tmp/coverage/ + path: /tmp/coverage + skip-if-does-not-exist: true # skip if there's no previous coverage to download + + - name: Upload coverage data + uses: ./.github/actions/upload + with: + name: coverage-data-artifact + path: /tmp/coverage diff --git a/.github/actions/upload/action.yml b/.github/actions/upload/action.yml new file mode 100644 index 0000000000..28e7d1fb1a --- /dev/null +++ b/.github/actions/upload/action.yml @@ -0,0 +1,51 @@ +name: "Upload an artifact" +description: "Custom upload action" +inputs: + name: + description: "Artifact name" + required: true + path: + description: "A directory or file to upload" + required: true + +runs: + using: "composite" + steps: + - name: Prepare artifact + shell: bash -euxo pipefail {0} + env: + SOURCE: ${{ inputs.path }} + ARCHIVE: /tmp/uploads/${{ inputs.name }}.tar.zst + run: | + mkdir -p $(dirname $ARCHIVE) + + if [ -f ${ARCHIVE} ]; then + echo 2>&1 "File ${ARCHIVE} already exist. Something went wrong before" + exit 1 + fi + + ZSTD_NBTHREADS=0 + if [ -d ${SOURCE} ]; then + time tar -C ${SOURCE} -cf ${ARCHIVE} --zstd . + elif [ -f ${SOURCE} ]; then + time tar -cf ${ARCHIVE} --zstd ${SOURCE} + else + echo 2>&1 "${SOURCE} neither directory nor file, don't know how to handle it" + fi + + - name: Upload artifact + shell: bash -euxo pipefail {0} + env: + SOURCE: ${{ inputs.path }} + ARCHIVE: /tmp/uploads/${{ inputs.name }}.tar.zst + run: | + BUCKET=neon-github-public-dev + PREFIX=artifacts/${GITHUB_RUN_ID} + FILENAME=$(basename $ARCHIVE) + + FILESIZE=$(du -sh ${ARCHIVE} | cut -f1) + + time aws s3 mv --only-show-errors ${ARCHIVE} s3://${BUCKET}/${PREFIX}/${GITHUB_RUN_ATTEMPT}/${FILENAME} + + # Ref https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary + echo "[${FILENAME}](https://${BUCKET}.s3.amazonaws.com/${PREFIX}/${GITHUB_RUN_ATTEMPT}/${FILENAME}) ${FILESIZE}" >> ${GITHUB_STEP_SUMMARY} diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 312b4d1f46..0be108400c 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -3,8 +3,8 @@ name: Test and Deploy on: push: branches: - - main - - release + - main + - release pull_request: defaults: @@ -22,7 +22,8 @@ env: jobs: build-neon: - runs-on: [ self-hosted, Linux, k8s-runner ] + runs-on: dev + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:2746987948 strategy: fail-fast: false matrix: @@ -31,6 +32,7 @@ jobs: env: BUILD_TYPE: ${{ matrix.build_type }} + GIT_VERSION: ${{ github.sha }} steps: - name: Checkout @@ -123,6 +125,7 @@ jobs: mkdir -p /tmp/coverage/ mkdir -p /tmp/neon/test_bin/ + test_exe_paths=$( ${cov_prefix} cargo test $CARGO_FLAGS --message-format=json --no-run | jq -r '.executable | select(. != null)' @@ -145,25 +148,20 @@ jobs: - name: Install postgres binaries run: cp -a tmp_install /tmp/neon/pg_install - - name: Prepare neon artifact - run: ZSTD_NBTHREADS=0 tar -C /tmp/neon/ -cf ./neon.tar.zst --zstd . - - - name: Upload neon binaries - uses: actions/upload-artifact@v3 + - name: Upload Neon artifact + uses: ./.github/actions/upload with: - retention-days: 7 - if-no-files-found: error name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact - path: ./neon.tar.zst + path: /tmp/neon # XXX: keep this after the binaries.list is formed, so the coverage can properly work later - name: Merge and upload coverage data if: matrix.build_type == 'debug' uses: ./.github/actions/save-coverage-data - pg_regress-tests: - runs-on: [ self-hosted, Linux, k8s-runner ] + runs-on: dev + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:2746987948 needs: [ build-neon ] strategy: fail-fast: false @@ -190,7 +188,8 @@ jobs: uses: ./.github/actions/save-coverage-data other-tests: - runs-on: [ self-hosted, Linux, k8s-runner ] + runs-on: dev + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:2746987948 needs: [ build-neon ] strategy: fail-fast: false @@ -216,7 +215,8 @@ jobs: uses: ./.github/actions/save-coverage-data benchmarks: - runs-on: [ self-hosted, Linux, k8s-runner ] + runs-on: dev + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:2746987948 needs: [ build-neon ] strategy: fail-fast: false @@ -245,7 +245,8 @@ jobs: # while coverage is currently collected for the debug ones coverage-report: - runs-on: [ self-hosted, Linux, k8s-runner ] + runs-on: dev + container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rustlegacy:2746987948 needs: [ other-tests, pg_regress-tests ] strategy: fail-fast: false @@ -270,23 +271,17 @@ jobs: target/ key: v3-${{ runner.os }}-${{ matrix.build_type }}-cargo-${{ matrix.rust_toolchain }}-${{ hashFiles('Cargo.lock') }} - - name: Get Neon artifact for restoration - uses: actions/download-artifact@v3 + - name: Get Neon artifact + uses: ./.github/actions/download with: name: neon-${{ runner.os }}-${{ matrix.build_type }}-${{ matrix.rust_toolchain }}-artifact - path: ./neon-artifact/ + path: /tmp/neon - - name: Extract Neon artifact - run: | - mkdir -p /tmp/neon/ - tar -xf ./neon-artifact/neon.tar.zst -C /tmp/neon/ - rm -rf ./neon-artifact/ - - - name: Restore coverage data - uses: actions/download-artifact@v3 + - name: Get coverage artifact + uses: ./.github/actions/download with: name: coverage-data-artifact - path: /tmp/coverage/ + path: /tmp/coverage - name: Merge coverage data run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge @@ -324,40 +319,40 @@ jobs: }" trigger-e2e-tests: - runs-on: [ self-hosted, Linux, k8s-runner ] - needs: [ build-neon ] - steps: - - name: Set PR's status to pending and request a remote CI test - run: | - COMMIT_SHA=${{ github.event.pull_request.head.sha }} - COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}} + runs-on: [ self-hosted, Linux, k8s-runner ] + needs: [ build-neon ] + steps: + - name: Set PR's status to pending and request a remote CI test + run: | + COMMIT_SHA=${{ github.event.pull_request.head.sha }} + COMMIT_SHA=${COMMIT_SHA:-${{ github.sha }}} - REMOTE_REPO="${{ github.repository_owner }}/cloud" + REMOTE_REPO="${{ github.repository_owner }}/cloud" - curl -f -X POST \ - https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \ - -H "Accept: application/vnd.github.v3+json" \ - --user "${{ secrets.CI_ACCESS_TOKEN }}" \ - --data \ - "{ - \"state\": \"pending\", - \"context\": \"neon-cloud-e2e\", - \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\" - }" + curl -f -X POST \ + https://api.github.com/repos/${{ github.repository }}/statuses/$COMMIT_SHA \ + -H "Accept: application/vnd.github.v3+json" \ + --user "${{ secrets.CI_ACCESS_TOKEN }}" \ + --data \ + "{ + \"state\": \"pending\", + \"context\": \"neon-cloud-e2e\", + \"description\": \"[$REMOTE_REPO] Remote CI job is about to start\" + }" - curl -f -X POST \ - https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \ - -H "Accept: application/vnd.github.v3+json" \ - --user "${{ secrets.CI_ACCESS_TOKEN }}" \ - --data \ - "{ - \"ref\": \"main\", - \"inputs\": { - \"ci_job_name\": \"neon-cloud-e2e\", - \"commit_hash\": \"$COMMIT_SHA\", - \"remote_repo\": \"${{ github.repository }}\" - } - }" + curl -f -X POST \ + https://api.github.com/repos/$REMOTE_REPO/actions/workflows/testing.yml/dispatches \ + -H "Accept: application/vnd.github.v3+json" \ + --user "${{ secrets.CI_ACCESS_TOKEN }}" \ + --data \ + "{ + \"ref\": \"main\", + \"inputs\": { + \"ci_job_name\": \"neon-cloud-e2e\", + \"commit_hash\": \"$COMMIT_SHA\", + \"remote_repo\": \"${{ github.repository }}\" + } + }" docker-image: runs-on: [ self-hosted, Linux, k8s-runner ]