diff --git a/.github/renovate-controller-policy.md b/.github/renovate-controller-policy.md index c857b2f85..00163887a 100644 --- a/.github/renovate-controller-policy.md +++ b/.github/renovate-controller-policy.md @@ -35,6 +35,18 @@ For a growing app catalog, apply capacity improvements in this order: 5. Group application images that must move together and keep multi-service updates under tested maintainer review. 6. Persist the self-hosted cache with the SHA-pinned `actions/cache` step in `.github/workflows/renovate.yml`. +## Docker Hub 429 retry policy + +`.github/workflows/renovate-rate-limit-retry.yml` checks the latest full Renovate run once per hour. It dispatches another full scan only when the latest run is a completed failure whose logs explicitly contain a registry `429`, the failure is at least 55 minutes old, and fewer than six full scans were started that UTC day. Successful, incomplete, unrelated, and young failures are not retried. + +The retry controller does not keep a runner sleeping during Docker Hub's cooldown. The newly dispatched run becomes the latest run immediately, so later checks stop until that run completes. The daily cap bounds repeated registry failures. + +## Multi-service update policy + +Renovate cannot natively express a dependency rule where an auxiliary image may update only when a separate primary image also updates. `.github/renovate-primary-services.json` therefore identifies primary services, while `.github/scripts/renovate_sidecar_guard.py` closes PRs that change only auxiliary images in a multi-service app. The guard also removes the closed PR's temporary branch only when it belongs to this repository and uses a known Renovate branch prefix; fork branches are never deleted. + +This is post-creation suppression, not a guarantee that Renovate never creates a temporary branch or PR. Continue disabling known auxiliary packages in `.github/renovate-docker.json` where a stable path/package rule is available. Single-service apps using the same image name are unaffected because those rules and the runtime guard are scoped by app path and Compose service count. + The workflow stores Renovate's public package lookup cache and repository extraction cache under `/tmp/renovate-cache`. Private package caching remains disabled. Cache writes come only from the scheduled or manually dispatched trusted workflow. GitHub Actions cache entries are immutable, so each run uses a unique key and restores the newest compatible prefix. The configuration hash separates policy generations, while the broader fallback retains public package lookup data after policy changes. Monitor the reported directory size and repository cache inventory to avoid churn against GitHub's default cache quota. diff --git a/.github/scripts/renovate_retry_gate.py b/.github/scripts/renovate_retry_gate.py new file mode 100644 index 000000000..fdece3d41 --- /dev/null +++ b/.github/scripts/renovate_retry_gate.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from datetime import datetime, timedelta, timezone +from pathlib import Path + + +MIN_RETRY_AGE = timedelta(minutes=55) +MAX_DAILY_ATTEMPTS = 6 +DOCKER_HUB_429 = re.compile( + r"(?:status\s+code\s+429|" + r"\b429\b.{0,80}(?:too many requests|rate.?limit)|" + r"(?:too many requests|rate.?limit).{0,80}\b429\b)", + re.IGNORECASE | re.DOTALL, +) +DOCKER_HUB_MARKER = re.compile( + r"docker\s*hub|(?:index|registry-1)\.docker\.io|\bdocker\.io\b", + re.IGNORECASE, +) + + +def parse_timestamp(value: str) -> datetime: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def evaluate_retry(runs: list[dict], log_text: str, now: datetime) -> dict: + if not runs: + return {"decision": "skip", "reason": "no Renovate runs found"} + + latest = max(runs, key=lambda run: parse_timestamp(run["createdAt"])) + if latest.get("status") != "completed": + return {"decision": "skip", "reason": "latest Renovate run is not complete"} + if latest.get("conclusion") != "failure": + return {"decision": "skip", "reason": "latest Renovate run did not fail"} + if not (DOCKER_HUB_429.search(log_text) and DOCKER_HUB_MARKER.search(log_text)): + return {"decision": "skip", "reason": "latest failure is not a confirmed Docker Hub 429"} + + completed_at = parse_timestamp(latest.get("updatedAt") or latest["createdAt"]) + now = now.astimezone(timezone.utc) + if now - completed_at < MIN_RETRY_AGE: + return {"decision": "wait", "reason": "429 cooldown has not reached 55 minutes"} + + attempts_today = sum( + 1 + for run in runs + if parse_timestamp(run["createdAt"]).date() == now.date() + ) + if attempts_today >= MAX_DAILY_ATTEMPTS: + return {"decision": "limit", "reason": "daily full-scan attempt limit reached"} + + return { + "decision": "retry", + "reason": "latest full scan failed with a mature Docker Hub 429", + "run_id": latest.get("databaseId"), + "attempts_today": attempts_today, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--runs", required=True, type=Path) + parser.add_argument("--logs", required=True, type=Path) + parser.add_argument("--now") + args = parser.parse_args() + + runs = json.loads(args.runs.read_text(encoding="utf-8")) + log_text = args.logs.read_text(encoding="utf-8", errors="replace") + now = parse_timestamp(args.now) if args.now else datetime.now(timezone.utc) + print(json.dumps(evaluate_retry(runs, log_text, now), ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/test_renovate_app_version.py b/.github/scripts/test_renovate_app_version.py index 4ae8f1069..aa8b1b572 100644 --- a/.github/scripts/test_renovate_app_version.py +++ b/.github/scripts/test_renovate_app_version.py @@ -8,6 +8,7 @@ import unittest SCRIPT_PATH = pathlib.Path(__file__).with_name("renovate_app_version.py") +RETRY_SCRIPT_PATH = pathlib.Path(__file__).with_name("renovate_retry_gate.py") REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] DOCKER_CONFIG = REPO_ROOT / ".github" / "renovate-docker.json" @@ -21,11 +22,83 @@ def load_module(): return module +def load_retry_module(): + spec = importlib.util.spec_from_file_location("renovate_retry_gate", RETRY_SCRIPT_PATH) + if spec is None or spec.loader is None: + raise RuntimeError(f"unable to load {RETRY_SCRIPT_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def compose(images): return {"services": {name: {"image": image} for name, image in images.items()}} class RenovateAppVersionTests(unittest.TestCase): + def test_retry_gate_retries_only_mature_docker_hub_429_failures(self): + module = load_retry_module() + now = module.parse_timestamp("2026-07-12T06:00:00Z") + runs = [ + { + "databaseId": 10, + "status": "completed", + "conclusion": "failure", + "createdAt": "2026-07-12T04:50:00Z", + "updatedAt": "2026-07-12T05:00:00Z", + } + ] + + decision = module.evaluate_retry(runs, "HTTP 429 Too Many Requests from index.docker.io", now) + + self.assertEqual("retry", decision["decision"]) + + def test_retry_gate_rejects_non_rate_limit_failures_and_retry_storms(self): + module = load_retry_module() + now = module.parse_timestamp("2026-07-12T06:00:00Z") + base_run = { + "status": "completed", + "conclusion": "failure", + "createdAt": "2026-07-12T04:50:00Z", + "updatedAt": "2026-07-12T05:00:00Z", + } + + non_429 = module.evaluate_retry( + [{"databaseId": 10, **base_run}], "ordinary configuration failure", now + ) + too_soon = module.evaluate_retry( + [{"databaseId": 10, **base_run, "updatedAt": "2026-07-12T05:30:00Z"}], + "Docker Hub status code 429", + now, + ) + too_many = module.evaluate_retry( + [ + {"databaseId": index, **base_run, "createdAt": f"2026-07-12T0{index}:00:00Z"} + for index in range(1, 7) + ], + "registry-1.docker.io status code 429", + now, + ) + + self.assertEqual("skip", non_429["decision"]) + self.assertEqual("wait", too_soon["decision"]) + self.assertEqual("limit", too_many["decision"]) + + def test_retry_workflow_and_sidecar_branch_cleanup_are_guarded(self): + retry_workflow = ( + REPO_ROOT / ".github" / "workflows" / "renovate-rate-limit-retry.yml" + ).read_text(encoding="utf-8") + sidecar_workflow = ( + REPO_ROOT / ".github" / "workflows" / "renovate-sidecar-guard.yml" + ).read_text(encoding="utf-8") + + self.assertIn("17 * * * *", retry_workflow) + self.assertIn("actions: write", retry_workflow) + self.assertIn("renovate_retry_gate.py", retry_workflow) + self.assertIn("automatic retry after Docker Hub 429", retry_workflow) + self.assertIn('[[ "$head_repo" == "$REPO" ]]', sidecar_workflow) + self.assertIn('selfhosted-renovate/*|renovate/*', sidecar_workflow) + self.assertIn("git/refs/heads/$head_ref", sidecar_workflow) def test_image_tag_strips_digest_from_tagged_image(self): module = load_module() diff --git a/.github/workflows/renovate-rate-limit-retry.yml b/.github/workflows/renovate-rate-limit-retry.yml new file mode 100644 index 000000000..5075be2fa --- /dev/null +++ b/.github/workflows/renovate-rate-limit-retry.yml @@ -0,0 +1,57 @@ +name: Retry Renovate after Docker Hub rate limit + +on: + schedule: + - cron: "17 * * * *" + workflow_dispatch: + +permissions: + contents: read + actions: write + +concurrency: + group: renovate-rate-limit-retry + cancel-in-progress: true + +jobs: + evaluate: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Inspect latest full Renovate run + id: gate + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + gh run list --repo "$REPO" --workflow renovate.yml --limit 50 \ + --json databaseId,status,conclusion,createdAt,updatedAt > /tmp/renovate-runs.json + + latest_run_id=$(jq -r 'sort_by(.createdAt) | last | .databaseId // empty' /tmp/renovate-runs.json) + : > /tmp/renovate-latest.log + if [[ -n "$latest_run_id" ]]; then + gh api "repos/$REPO/actions/runs/$latest_run_id/jobs?per_page=100" \ + --jq '.jobs[].id' | while read -r job_id; do + gh api "repos/$REPO/actions/jobs/$job_id/logs" >> /tmp/renovate-latest.log || true + done + fi + + python3 .github/scripts/renovate_retry_gate.py \ + --runs /tmp/renovate-runs.json \ + --logs /tmp/renovate-latest.log > /tmp/renovate-retry.json + cat /tmp/renovate-retry.json + echo "decision=$(jq -r '.decision' /tmp/renovate-retry.json)" >> "$GITHUB_OUTPUT" + + - name: Dispatch full Renovate scan + if: steps.gate.outputs.decision == 'retry' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + run: | + gh workflow run renovate.yml \ + --repo "$REPO" \ + --ref localApps \ + -f manual-trigger="automatic retry after Docker Hub 429" diff --git a/.github/workflows/renovate-sidecar-guard.yml b/.github/workflows/renovate-sidecar-guard.yml index 62204536a..07584daeb 100644 --- a/.github/workflows/renovate-sidecar-guard.yml +++ b/.github/workflows/renovate-sidecar-guard.yml @@ -13,7 +13,7 @@ on: type: string permissions: - contents: read + contents: write pull-requests: write jobs: @@ -88,3 +88,18 @@ jobs: gh pr close "$pr_number" \ --repo "${{ github.repository }}" + + REPO="${{ github.repository }}" + pr_head=$(gh api "repos/$REPO/pulls/$pr_number" \ + --jq '[.head.repo.full_name, .head.ref] | @tsv') + IFS=$'\t' read -r head_repo head_ref <<< "$pr_head" + + if [[ "$head_repo" == "$REPO" ]]; then + case "$head_ref" in + selfhosted-renovate/*|renovate/*) + if ! gh api --method DELETE "repos/$REPO/git/refs/heads/$head_ref"; then + echo "::notice::Renovate branch was already removed or could not be deleted: $head_ref" + fi + ;; + esac + fi