diff --git a/.github/scripts/push_with_image_map.py b/.github/scripts/push_with_image_map.py index c68f6ad407..53f83379ae 100644 --- a/.github/scripts/push_with_image_map.py +++ b/.github/scripts/push_with_image_map.py @@ -11,12 +11,27 @@ try: except json.JSONDecodeError as e: raise ValueError("Failed to parse IMAGE_MAP as JSON") from e -for source, targets in parsed_image_map.items(): - for target in targets: - cmd = ["docker", "buildx", "imagetools", "create", "-t", target, source] - print(f"Running: {' '.join(cmd)}") - result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) +failures = [] - if result.returncode != 0: - print(f"Error: {result.stdout}") - raise RuntimeError(f"Command failed: {' '.join(cmd)}") +pending = [(source, target) for source, targets in parsed_image_map.items() for target in targets] + +while len(pending) > 0: + if len(failures) > 10: + print("Error: more than 10 failures!") + for failure in failures: + print(f'"{failure[0]}" failed with the following output:') + print(failure[1]) + raise RuntimeError("Retry limit reached.") + + source, target = pending.pop(0) + cmd = ["docker", "buildx", "imagetools", "create", "-t", target, source] + print(f"Running: {' '.join(cmd)}") + result = subprocess.run(cmd, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if result.returncode != 0: + failures.append((" ".join(cmd), result.stdout)) + pending.append((source, target)) + +if len(failures) > 0 and (github_output := os.getenv("GITHUB_OUTPUT")): + with open(github_output, "a") as f: + f.write("slack_notify=true\n") diff --git a/.github/workflows/_push-to-container-registry.yml b/.github/workflows/_push-to-container-registry.yml index 9a7da612d4..9b3ad0fdbb 100644 --- a/.github/workflows/_push-to-container-registry.yml +++ b/.github/workflows/_push-to-container-registry.yml @@ -104,6 +104,18 @@ jobs: password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - name: Copy docker images to target registries + id: push run: python3 .github/scripts/push_with_image_map.py env: IMAGE_MAP: ${{ inputs.image-map }} + + - name: Notify Slack if container image pushing fails + if: steps.push.outputs.slack_notify == 'true' || failure() + uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 + with: + method: chat.postMessage + token: ${{ secrets.SLACK_BOT_TOKEN }} + payload: | + channel: ${{ vars.SLACK_ON_CALL_DEVPROD_STREAM }} + text: | + Pushing container images failed in <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run>