From e52e93797fbb930da8cf7139e150d748920182f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JC=20Gr=C3=BCnhage?= Date: Wed, 19 Feb 2025 13:34:41 +0100 Subject: [PATCH] refactor(ci): use variables for AWS account IDs (#10886) ## Problem Our AWS account IDs are copy-pasted all over the place. A wrong paste might only be caught late if we hardcode them, but will get flagged instantly by actionlint if we access them from github actions variables. Resolves https://github.com/neondatabase/neon/issues/10787, follow-up for https://github.com/neondatabase/neon/pull/10613. ## Summary of changes Access AWS account IDs using Github Actions variables. --- .github/actionlint.yml | 3 +++ .../workflows/_push-to-container-registry.yml | 2 +- .github/workflows/build_and_test.yml | 23 +++++++++++-------- .../build_and_test_with_sanitizers.yml | 2 +- .github/workflows/pin-build-tools-image.yml | 6 ++--- scripts/generate_image_maps.py | 7 ++++-- 6 files changed, 26 insertions(+), 17 deletions(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 5114517e7f..1e6c2d0aa2 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -29,3 +29,6 @@ config-variables: - SLACK_ON_CALL_STORAGE_STAGING_STREAM - SLACK_CICD_CHANNEL_ID - SLACK_STORAGE_CHANNEL_ID + - NEON_DEV_AWS_ACCOUNT_ID + - NEON_PROD_AWS_ACCOUNT_ID + - AWS_ECR_REGION diff --git a/.github/workflows/_push-to-container-registry.yml b/.github/workflows/_push-to-container-registry.yml index 3c97c8a67a..c938f62ad5 100644 --- a/.github/workflows/_push-to-container-registry.yml +++ b/.github/workflows/_push-to-container-registry.yml @@ -2,7 +2,7 @@ name: Push images to Container Registry on: workflow_call: inputs: - # Example: {"docker.io/neondatabase/neon:13196061314":["369495373322.dkr.ecr.eu-central-1.amazonaws.com/neon:13196061314","neoneastus2.azurecr.io/neondatabase/neon:13196061314"]} + # Example: {"docker.io/neondatabase/neon:13196061314":["${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/neon:13196061314","neoneastus2.azurecr.io/neondatabase/neon:13196061314"]} image-map: description: JSON map of images, mapping from a source image to an array of target images that should be pushed. required: true diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index d9bf094aa9..f08280e112 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -68,7 +68,7 @@ jobs: tag: needs: [ check-permissions ] runs-on: [ self-hosted, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned + container: ${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/base:pinned outputs: build-tag: ${{steps.build-tag.outputs.tag}} @@ -859,14 +859,17 @@ jobs: BRANCH: "${{ github.ref_name }}" DEV_ACR: "${{ vars.AZURE_DEV_REGISTRY_NAME }}" PROD_ACR: "${{ vars.AZURE_PROD_REGISTRY_NAME }}" + DEV_AWS: "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}" + PROD_AWS: "${{ vars.NEON_PROD_AWS_ACCOUNT_ID }}" + AWS_REGION: "${{ vars.AWS_ECR_REGION }}" push-neon-image-dev: needs: [ generate-image-maps, neon-image ] uses: ./.github/workflows/_push-to-container-registry.yml with: image-map: '${{ needs.generate-image-maps.outputs.neon-dev }}' - aws-region: eu-central-1 - aws-account-ids: "369495373322" + aws-region: ${{ vars.AWS_ECR_REGION }} + aws-account-ids: "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}" azure-client-id: ${{ vars.AZURE_DEV_CLIENT_ID }} azure-subscription-id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }} azure-tenant-id: ${{ vars.AZURE_TENANT_ID }} @@ -881,8 +884,8 @@ jobs: uses: ./.github/workflows/_push-to-container-registry.yml with: image-map: '${{ needs.generate-image-maps.outputs.compute-dev }}' - aws-region: eu-central-1 - aws-account-ids: "369495373322" + aws-region: ${{ vars.AWS_ECR_REGION }} + aws-account-ids: "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}" azure-client-id: ${{ vars.AZURE_DEV_CLIENT_ID }} azure-subscription-id: ${{ vars.AZURE_DEV_SUBSCRIPTION_ID }} azure-tenant-id: ${{ vars.AZURE_TENANT_ID }} @@ -898,8 +901,8 @@ jobs: uses: ./.github/workflows/_push-to-container-registry.yml with: image-map: '${{ needs.generate-image-maps.outputs.neon-prod }}' - aws-region: eu-central-1 - aws-account-ids: "093970136003" + aws-region: ${{ vars.AWS_ECR_REGION }} + aws-account-ids: "${{ vars.NEON_PROD_AWS_ACCOUNT_ID }}" azure-client-id: ${{ vars.AZURE_PROD_CLIENT_ID }} azure-subscription-id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }} azure-tenant-id: ${{ vars.AZURE_TENANT_ID }} @@ -915,8 +918,8 @@ jobs: uses: ./.github/workflows/_push-to-container-registry.yml with: image-map: '${{ needs.generate-image-maps.outputs.compute-prod }}' - aws-region: eu-central-1 - aws-account-ids: "093970136003" + aws-region: ${{ vars.AWS_ECR_REGION }} + aws-account-ids: "${{ vars.NEON_PROD_AWS_ACCOUNT_ID }}" azure-client-id: ${{ vars.AZURE_PROD_CLIENT_ID }} azure-subscription-id: ${{ vars.AZURE_PROD_SUBSCRIPTION_ID }} azure-tenant-id: ${{ vars.AZURE_TENANT_ID }} @@ -1029,7 +1032,7 @@ jobs: statuses: write contents: write runs-on: [ self-hosted, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest + container: ${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/ansible:latest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/build_and_test_with_sanitizers.yml b/.github/workflows/build_and_test_with_sanitizers.yml index 2bc938509f..e40b02b5d2 100644 --- a/.github/workflows/build_and_test_with_sanitizers.yml +++ b/.github/workflows/build_and_test_with_sanitizers.yml @@ -27,7 +27,7 @@ env: jobs: tag: runs-on: [ self-hosted, small ] - container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/base:pinned + container: ${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/base:pinned outputs: build-tag: ${{steps.build-tag.outputs.tag}} diff --git a/.github/workflows/pin-build-tools-image.yml b/.github/workflows/pin-build-tools-image.yml index 626de2b0e0..8861c1f093 100644 --- a/.github/workflows/pin-build-tools-image.yml +++ b/.github/workflows/pin-build-tools-image.yml @@ -78,7 +78,7 @@ jobs: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - aws-region: eu-central-1 + aws-region: ${{ vars.AWS_ECR_REGION }} role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} role-duration-seconds: 3600 @@ -104,12 +104,12 @@ jobs: tags=() tags+=("-t" "neondatabase/build-tools:${TO_TAG}-${debian_version}") - tags+=("-t" "369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${TO_TAG}-${debian_version}") + tags+=("-t" "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/build-tools:${TO_TAG}-${debian_version}") tags+=("-t" "neoneastus2.azurecr.io/neondatabase/build-tools:${TO_TAG}-${debian_version}") if [ "${debian_version}" == "${DEFAULT_DEBIAN_VERSION}" ]; then tags+=("-t" "neondatabase/build-tools:${TO_TAG}") - tags+=("-t" "369495373322.dkr.ecr.eu-central-1.amazonaws.com/build-tools:${TO_TAG}") + tags+=("-t" "${{ vars.NEON_DEV_AWS_ACCOUNT_ID }}.dkr.ecr.${{ vars.AWS_ECR_REGION }}.amazonaws.com/build-tools:${TO_TAG}") tags+=("-t" "neoneastus2.azurecr.io/neondatabase/build-tools:${TO_TAG}") fi diff --git a/scripts/generate_image_maps.py b/scripts/generate_image_maps.py index a2f553d290..915eb33673 100644 --- a/scripts/generate_image_maps.py +++ b/scripts/generate_image_maps.py @@ -6,6 +6,9 @@ build_tag = os.environ["BUILD_TAG"] branch = os.environ["BRANCH"] dev_acr = os.environ["DEV_ACR"] prod_acr = os.environ["PROD_ACR"] +dev_aws = os.environ["DEV_AWS"] +prod_aws = os.environ["PROD_AWS"] +aws_region = os.environ["AWS_REGION"] components = { "neon": ["neon"], @@ -24,11 +27,11 @@ components = { registries = { "dev": [ "docker.io/neondatabase", - "369495373322.dkr.ecr.eu-central-1.amazonaws.com", + f"{dev_aws}.dkr.ecr.{aws_region}.amazonaws.com", f"{dev_acr}.azurecr.io/neondatabase", ], "prod": [ - "093970136003.dkr.ecr.eu-central-1.amazonaws.com", + f"{prod_aws}.dkr.ecr.{aws_region}.amazonaws.com", f"{prod_acr}.azurecr.io/neondatabase", ], }