From ea84ec357fa4caa5a48ec65a0aab9e37d1a9fda4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JC=20Gr=C3=BCnhage?= Date: Tue, 7 Jan 2025 11:36:05 +0100 Subject: [PATCH] Split promote-images into promote-images-dev and promote-images-prod (#10267) ## Problem `trigger-e2e-tests` waits half an hour before starting to run. Nearly half of that time can be saved by promoting images before tests on them are complete, so the e2e tests can run in parallel. On `main` and `release{,-proxy,-compute}`, `promote-images` updates `latest` and pushes things to prod ecr, so we want to run `promote-images` only after `test-images` is done, but on other branches, there is no harm in promoting images that aren't tested yet. ## Summary of changes To promote images into dev container registries sooner, `promote-images` is split into `promote-images-dev` and `promote-images-prod`. The former pushes to dev container registries, the latter to prod ones. The latter also waits for `test-images`, while the former doesn't. This allows to run `trigger-e2e-tests` sooner. --- .github/workflows/actionlint.yml | 2 +- .github/workflows/build_and_test.yml | 35 +++++++++++++++++++------ .github/workflows/trigger-e2e-tests.yml | 8 +++--- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index 85cfe7446e..0e53830040 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -33,7 +33,7 @@ jobs: # SC2086 - Double quote to prevent globbing and word splitting. - https://www.shellcheck.net/wiki/SC2086 SHELLCHECK_OPTS: --exclude=SC2046,SC2086 with: - fail_on_error: true + fail_level: error filter_mode: nofilter level: error diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 12b1ac98ac..5c2b397c82 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -538,7 +538,7 @@ jobs: trigger-e2e-tests: if: ${{ !github.event.pull_request.draft || contains( github.event.pull_request.labels.*.name, 'run-e2e-tests-in-draft') || github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' }} - needs: [ check-permissions, promote-images, tag ] + needs: [ check-permissions, promote-images-dev, tag ] uses: ./.github/workflows/trigger-e2e-tests.yml secrets: inherit @@ -930,8 +930,8 @@ jobs: docker compose -f ./docker-compose/docker-compose.yml logs || 0 docker compose -f ./docker-compose/docker-compose.yml down - promote-images: - needs: [ check-permissions, tag, test-images, vm-compute-node-image ] + promote-images-dev: + needs: [ check-permissions, tag, vm-compute-node-image ] runs-on: ubuntu-22.04 permissions: @@ -965,6 +965,25 @@ jobs: neondatabase/vm-compute-node-${version}:${{ needs.tag.outputs.build-tag }} done + promote-images-prod: + needs: [ check-permissions, tag, test-images, vm-compute-node-image ] + runs-on: ubuntu-22.04 + if: github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' + + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: read + + env: + VERSIONS: v14 v15 v16 v17 + + steps: + - uses: docker/login-action@v3 + with: + username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} + password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} + - name: Add latest tag to images if: github.ref_name == 'main' run: | @@ -1010,7 +1029,7 @@ jobs: push-to-acr-dev: if: github.ref_name == 'main' - needs: [ tag, promote-images ] + needs: [ tag, promote-images-dev ] uses: ./.github/workflows/_push-to-acr.yml with: client_id: ${{ vars.AZURE_DEV_CLIENT_ID }} @@ -1022,7 +1041,7 @@ jobs: push-to-acr-prod: if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' - needs: [ tag, promote-images ] + needs: [ tag, promote-images-prod ] uses: ./.github/workflows/_push-to-acr.yml with: client_id: ${{ vars.AZURE_PROD_CLIENT_ID }} @@ -1112,7 +1131,7 @@ jobs: exit 1 deploy: - needs: [ check-permissions, promote-images, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait, push-to-acr-dev, push-to-acr-prod ] + needs: [ check-permissions, promote-images-prod, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait, push-to-acr-dev, push-to-acr-prod ] # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod` if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute') && !failure() && !cancelled() permissions: @@ -1333,7 +1352,7 @@ jobs: done pin-build-tools-image: - needs: [ build-build-tools-image, promote-images, build-and-test-locally ] + needs: [ build-build-tools-image, promote-images-prod, build-and-test-locally ] if: github.ref_name == 'main' uses: ./.github/workflows/pin-build-tools-image.yml with: @@ -1356,7 +1375,7 @@ jobs: - build-and-test-locally - check-codestyle-python - check-codestyle-rust - - promote-images + - promote-images-dev - test-images - trigger-custom-extensions-build-and-wait runs-on: ubuntu-22.04 diff --git a/.github/workflows/trigger-e2e-tests.yml b/.github/workflows/trigger-e2e-tests.yml index 70c2e8549f..31696248b0 100644 --- a/.github/workflows/trigger-e2e-tests.yml +++ b/.github/workflows/trigger-e2e-tests.yml @@ -68,7 +68,7 @@ jobs: GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} TAG: ${{ needs.tag.outputs.build-tag }} steps: - - name: Wait for `promote-images` job to finish + - name: Wait for `promote-images-dev` job to finish # It's important to have a timeout here, the script in the step can run infinitely timeout-minutes: 60 run: | @@ -79,17 +79,17 @@ jobs: # For PRs we use the run id as the tag BUILD_AND_TEST_RUN_ID=${TAG} while true; do - conclusion=$(gh run --repo ${GITHUB_REPOSITORY} view ${BUILD_AND_TEST_RUN_ID} --json jobs --jq '.jobs[] | select(.name == "promote-images") | .conclusion') + conclusion=$(gh run --repo ${GITHUB_REPOSITORY} view ${BUILD_AND_TEST_RUN_ID} --json jobs --jq '.jobs[] | select(.name == "promote-images-dev") | .conclusion') case "$conclusion" in success) break ;; failure | cancelled | skipped) - echo "The 'promote-images' job didn't succeed: '${conclusion}'. Exiting..." + echo "The 'promote-images-dev' job didn't succeed: '${conclusion}'. Exiting..." exit 1 ;; *) - echo "The 'promote-images' hasn't succeed yet. Waiting..." + echo "The 'promote-images-dev' hasn't succeed yet. Waiting..." sleep 60 ;; esac