Fix Renovate app-version automation gates

This commit is contained in:
okxlin
2026-11-06 19:44:00 +08:00
parent 74a54186ef
commit eb39f0f45c
6 changed files with 590 additions and 204 deletions
@@ -114,6 +114,8 @@ def select_sources(
raise ValueError("base and head refs must be full Git commit SHAs")
grouped: dict[str, list[str]] = {}
for raw_path in paths:
if not raw_path.endswith("/docker-compose.yml"):
continue
app, version = parse_compose_path(raw_path)
if version in ROLLING_VERSIONS or not git_path_exists(base_ref, raw_path):
continue
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
import copy
import json
import re
import subprocess
import sys
from pathlib import Path
from typing import Any
import yaml
class GateReject(Exception):
pass
def run_git(*args: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
["git", *args], check=False, text=True, capture_output=True
)
def tree_files(ref: str, directory: str) -> dict[str, tuple[str, str]]:
result = run_git("ls-tree", "-r", "-z", ref, "--", directory)
if result.returncode != 0:
raise RuntimeError(result.stderr.strip() or f"unable to read tree {ref}:{directory}")
prefix = directory.rstrip("/") + "/"
files: dict[str, tuple[str, str]] = {}
for entry in result.stdout.split("\0"):
if not entry:
continue
metadata, path = entry.split("\t", 1)
mode, object_type, oid = metadata.split(" ", 2)
if object_type == "blob" and path.startswith(prefix):
files[path[len(prefix) :]] = (mode, oid)
return files
def git_show(ref: str, path: str) -> str:
result = run_git("show", f"{ref}:{path}")
if result.returncode != 0:
raise GateReject(f"cannot read Compose file {ref}:{path}")
return result.stdout
def read_compose(path: str, content: str) -> tuple[dict[str, Any], dict[str, Any]]:
try:
data = yaml.safe_load(content) or {}
except yaml.YAMLError as exc:
raise GateReject(f"invalid Compose YAML: {path}: {exc}") from exc
if not isinstance(data, dict):
raise GateReject(f"invalid Compose mapping: {path}")
services = data.get("services")
if not isinstance(services, dict) or len(services) != 1:
raise GateReject(f"Compose must contain exactly one service: {path}")
return data, services
def digest_reference(image: object) -> tuple[str, str] | None:
if not isinstance(image, str):
return None
match = re.fullmatch(r"([^@\s]+)@sha256:([0-9a-fA-F]{64})", image)
if match is None:
return None
return match.group(1), match.group(2).lower()
def verify_compose_pair(
old_path: str, old_raw: str, new_path: str, new_raw: str
) -> bool:
old_data, old_services = read_compose(old_path, old_raw)
new_data, new_services = read_compose(new_path, new_raw)
old_service = next(iter(old_services.values()))
new_service = next(iter(new_services.values()))
if not isinstance(old_service, dict) or not isinstance(new_service, dict):
raise GateReject(f"Compose service must be a mapping: {new_path}")
old_image = old_service.get("image")
new_image = new_service.get("image")
if not isinstance(old_image, str) or not isinstance(new_image, str) or old_image == new_image:
raise GateReject(f"Compose service image must change: {new_path}")
old_normalized = copy.deepcopy(old_data)
new_normalized = copy.deepcopy(new_data)
old_normalized.pop("version", None)
new_normalized.pop("version", None)
for service in old_normalized["services"].values():
if isinstance(service, dict):
service.pop("image", None)
for service in new_normalized["services"].values():
if isinstance(service, dict):
service.pop("image", None)
if old_normalized != new_normalized:
raise GateReject(f"Compose changed beyond its service image: {new_path}")
old_digest = digest_reference(old_image)
new_digest = digest_reference(new_image)
return bool(
old_digest is not None
and new_digest is not None
and old_digest[0] == new_digest[0]
and old_digest[1] != new_digest[1]
)
def app_names(files: list[dict[str, Any]]) -> set[str]:
apps: set[str] = set()
for item in files:
for path in (item.get("filename"), item.get("previous_filename")):
if not path:
continue
match = re.fullmatch(r"apps/([^/]+)/.+", path)
if match is None:
raise GateReject(f"changed file outside apps/: {path}")
apps.add(match.group(1))
if len(apps) != 1:
raise GateReject(f"PR must affect exactly one app, got {', '.join(sorted(apps))}")
return apps
def verify_added_version_package(
base_sha: str, head_sha: str, app: str, files: list[dict[str, Any]]
) -> bool:
target_dirs: set[str] = set()
changed_paths: set[str] = set()
for item in files:
path = item.get("filename", "")
parts = path.split("/")
if (
item.get("status") != "added"
or item.get("previous_filename")
or len(parts) < 4
or parts[0] != "apps"
or parts[1] != app
or any(part in {"", ".", ".."} for part in parts)
):
raise GateReject("new version packages must contain only added files under one app directory")
target_dirs.add("/".join(parts[:3]))
changed_paths.add(path)
if len(target_dirs) != 1:
raise GateReject("PR must add exactly one version directory")
target_dir = next(iter(target_dirs))
target_tree = tree_files(head_sha, target_dir)
if not target_tree or {f"{target_dir}/{path}" for path in target_tree} != changed_paths:
raise GateReject("added version directory must exactly match the PR file list")
compose_path = f"{target_dir}/docker-compose.yml"
if "docker-compose.yml" not in target_tree:
raise GateReject("added version directory must include docker-compose.yml")
base_app = tree_files(base_sha, f"apps/{app}")
source_versions = sorted(
path.split("/", 1)[0]
for path in base_app
if path.count("/") == 1 and path.endswith("/docker-compose.yml")
)
for source_version in source_versions:
source_dir = f"apps/{app}/{source_version}"
source_tree = tree_files(base_sha, source_dir)
if set(source_tree) != set(target_tree):
continue
if any(source_tree[path][0] != target_tree[path][0] for path in source_tree):
continue
if any(
source_tree[path][1] != target_tree[path][1]
for path in source_tree
if path != "docker-compose.yml"
):
continue
old_compose = git_show(base_sha, f"{source_dir}/docker-compose.yml")
new_compose = git_show(head_sha, compose_path)
verify_compose_pair(
f"{source_dir}/docker-compose.yml", old_compose, compose_path, new_compose
)
print(
f"verified added version package {target_dir} as an image-only copy of {source_dir}"
)
return False
raise GateReject("added version directory does not exactly copy a base package except for image")
def verify_existing_compose_updates(
base_sha: str, head_sha: str, app: str, files: list[dict[str, Any]]
) -> bool:
compose_files: list[tuple[str, str]] = []
for item in files:
filename = item.get("filename", "")
previous = item.get("previous_filename")
if not filename.endswith("/docker-compose.yml") and not (
previous and previous.endswith("/docker-compose.yml")
):
raise GateReject(f"whitelisted PR contains a non-Compose file: {filename}")
compose_files.append((previous or filename, filename))
if not compose_files:
raise GateReject("PR must change at least one docker-compose.yml")
digest_only = True
for old_path, new_path in compose_files:
old_match = re.fullmatch(r"apps/([^/]+)/.+/docker-compose\.yml", old_path)
new_match = re.fullmatch(r"apps/([^/]+)/.+/docker-compose\.yml", new_path)
if not old_match or not new_match or old_match.group(1) != app or new_match.group(1) != app:
raise GateReject(f"invalid Compose path for app {app}: {new_path}")
old_raw = git_show(base_sha, old_path)
new_raw = git_show(head_sha, new_path)
if not verify_compose_pair(old_path, old_raw, new_path, new_raw):
digest_only = False
print(f"verified {new_path}: image-only update")
return digest_only
def verify_shape(base_sha: str, head_sha: str, files: list[dict[str, Any]]) -> dict[str, Any]:
if not files:
raise GateReject("PR does not change files")
apps = app_names(files)
app = next(iter(apps))
if all(item.get("status") == "added" for item in files):
digest_only = verify_added_version_package(base_sha, head_sha, app, files)
return {"app": app, "digest_only": digest_only, "kind": "added-version-package"}
digest_only = verify_existing_compose_updates(base_sha, head_sha, app, files)
return {"app": app, "digest_only": digest_only, "kind": "compose-update"}
def main() -> int:
parser = argparse.ArgumentParser(description="Verify a whitelist Renovate PR image-only shape.")
parser.add_argument("--base-sha", required=True)
parser.add_argument("--head-sha", required=True)
parser.add_argument("--files-json", required=True, type=Path)
parser.add_argument("--output-json", required=True, type=Path)
args = parser.parse_args()
if not all(re.fullmatch(r"[0-9a-fA-F]{40,64}", sha) for sha in (args.base_sha, args.head_sha)):
raise ValueError("base and head must be full Git commit SHAs")
files = json.loads(args.files_json.read_text(encoding="utf-8"))
if not isinstance(files, list) or any(not isinstance(item, dict) for item in files):
raise ValueError("PR file list must be a JSON array of objects")
result = verify_shape(args.base_sha, args.head_sha, files)
args.output_json.parent.mkdir(parents=True, exist_ok=True)
args.output_json.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(f"single-app image-only update verified: {result['app']} ({result['kind']})")
print(f"digest_only={'true' if result['digest_only'] else 'false'}")
return 0
if __name__ == "__main__":
try:
raise SystemExit(main())
except GateReject as exc:
print(str(exc), file=sys.stderr)
raise SystemExit(42)
+90 -6
View File
@@ -266,6 +266,74 @@ class RenovateAppVersionTests(unittest.TestCase):
output_file.read_bytes().decode("utf-8").rstrip("\0").split("\0"),
)
def test_source_selector_ignores_base_only_non_compose_paths(self):
with tempfile.TemporaryDirectory(prefix="renovate-source-base-only-") as tmp:
repo = pathlib.Path(tmp)
app_compose = repo / "apps" / "demo" / "1.0.0" / "docker-compose.yml"
app_compose.parent.mkdir(parents=True)
app_compose.write_text(
"services:\n app:\n image: example/demo:1.0.0\n",
encoding="utf-8",
)
subprocess.run(["git", "init", "-q"], cwd=repo, check=True)
subprocess.run(["git", "config", "user.email", "test@example.com"], cwd=repo, check=True)
subprocess.run(["git", "config", "user.name", "Test User"], cwd=repo, check=True)
subprocess.run(["git", "add", "apps"], cwd=repo, check=True)
subprocess.run(["git", "commit", "-qm", "add demo source"], cwd=repo, check=True)
common_ref = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=repo, check=True, text=True, capture_output=True
).stdout.strip()
unrelated = repo / "apps" / "gomodel" / "0.2.2" / ".env.sample"
unrelated.parent.mkdir(parents=True)
unrelated.write_text("EXAMPLE=value\n", encoding="utf-8")
subprocess.run(["git", "add", "apps/gomodel"], cwd=repo, check=True)
subprocess.run(["git", "commit", "-qm", "add base-only package"], cwd=repo, check=True)
base_ref = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=repo, check=True, text=True, capture_output=True
).stdout.strip()
subprocess.run(["git", "checkout", "-q", "-b", "renovate", common_ref], cwd=repo, check=True)
app_compose.write_text(
"services:\n app:\n image: example/demo:1.1.0\n",
encoding="utf-8",
)
subprocess.run(["git", "add", "apps/demo"], cwd=repo, check=True)
subprocess.run(["git", "commit", "-qm", "update demo image"], cwd=repo, check=True)
head_ref = subprocess.run(
["git", "rev-parse", "HEAD"], cwd=repo, check=True, text=True, capture_output=True
).stdout.strip()
two_tree = subprocess.run(
["git", "diff", "--name-only", "-z", base_ref, head_ref],
cwd=repo, check=True, capture_output=True,
).stdout
three_dot = subprocess.run(
["git", "diff", "--name-only", "-z", f"{base_ref}...{head_ref}"],
cwd=repo, check=True, capture_output=True,
).stdout
self.assertIn(b"apps/gomodel/0.2.2/.env.sample", two_tree)
self.assertNotIn(b"apps/gomodel/0.2.2/.env.sample", three_dot)
changed_file_list = repo / "changed-files.zlist"
changed_file_list.write_bytes(two_tree)
policy_file = repo / "policy.json"
policy_file.write_text("{}\n", encoding="utf-8")
output_file = repo / "selected.zlist"
script = REPO_ROOT / ".github" / "scripts" / "renovate_app_version_sources.py"
result = subprocess.run(
[
sys.executable, str(script), "--base-ref", base_ref, "--head-ref", head_ref,
"--changed-files", str(changed_file_list), "--policy-file", str(policy_file),
"--output-file", str(output_file),
], cwd=repo, text=True, capture_output=True, check=False,
)
self.assertEqual(0, result.returncode, result.stderr)
self.assertEqual(
["apps/demo/1.0.0/docker-compose.yml"],
output_file.read_bytes().decode("utf-8").rstrip("\0").split("\0"),
)
def test_source_selector_fails_on_ambiguous_newest_version(self):
with tempfile.TemporaryDirectory(prefix="renovate-source-ambiguity-") as tmp:
repo = pathlib.Path(tmp)
@@ -1142,6 +1210,7 @@ class RenovateAppVersionTests(unittest.TestCase):
self.assertIn("renovate_app_version_sources.py", app_version)
self.assertIn('--head-ref "$(git rev-parse HEAD)"', app_version)
self.assertIn("selected-source-files.zlist", app_version)
self.assertIn('git diff --name-only -z "$base_ref...HEAD"', app_version)
self.assertIn('git cat-file -e "$BASE_REF:$changed_file"', app_version)
self.assertIn('git diff --cached --quiet "$BASE_REF" -- "$source_dir"', app_version)
self.assertIn('git restore --source="$BASE_REF" --staged --worktree -- "$changed_dir"', app_version)
@@ -1162,16 +1231,31 @@ class RenovateAppVersionTests(unittest.TestCase):
workflow = (
REPO_ROOT / ".github" / "workflows" / "renovate-automerge.yml"
).read_text(encoding="utf-8")
shape_helper = (
REPO_ROOT / ".github" / "scripts" / "renovate_automerge_shape.py"
).read_text(encoding="utf-8")
self.assertIn("def digest_reference(image):", workflow)
self.assertIn("renovate_automerge_shape.py", workflow)
self.assertIn("Verify Renovate PR identity", workflow)
self.assertIn('.user.login == "renovate[bot]"', workflow)
self.assertIn('.user.login == "app/renovate"', workflow)
self.assertIn('.head.repo.full_name == $repo', workflow)
self.assertIn('(.head.ref | startswith("renovate/"))', workflow)
self.assertIn('(.head.ref | startswith("selfhosted-renovate/"))', workflow)
self.assertIn("--files-json /tmp/renovate-pr-files.json", workflow)
self.assertIn('echo "head_sha=$HEAD_SHA"', workflow)
self.assertIn('--match-head-commit "$VERIFIED_HEAD_SHA"', workflow)
self.assertIn("PR head changed after shape verification", workflow)
self.assertIn('gh api "repos/$REPO/git/ref/heads/localApps"', workflow)
self.assertIn("localApps moved after shape verification", workflow)
self.assertIn(
're.fullmatch(r"([^@\\s]+)@sha256:([0-9a-fA-F]{64})", image)',
workflow,
shape_helper,
)
self.assertIn("old_digest[0] != new_digest[0]", workflow)
self.assertIn("old_digest[1] == new_digest[1]", workflow)
self.assertIn("filename_is_compose != previous_is_compose", workflow)
self.assertIn("digest_only=true", workflow)
self.assertIn("old_digest[0] == new_digest[0]", shape_helper)
self.assertIn("old_digest[1] != new_digest[1]", shape_helper)
self.assertIn('echo "digest_only=$digest_only"', workflow)
self.assertIn('"digest_only": digest_only', shape_helper)
self.assertIn("steps.shape.outputs.digest_only", workflow)
self.assertIn("Digest-only image update; app-version marker is not required", workflow)
self.assertIn("Update app version [skip ci]", workflow)
@@ -0,0 +1,139 @@
#!/usr/bin/env python3
from __future__ import annotations
import json
import pathlib
import shutil
import subprocess
import sys
import tempfile
import unittest
SCRIPT = pathlib.Path(__file__).with_name("renovate_automerge_shape.py")
def git(repo: pathlib.Path, *args: str) -> str:
result = subprocess.run(
["git", *args], cwd=repo, check=True, text=True, capture_output=True
)
return result.stdout.strip()
class RenovateAutomergeShapeTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory(prefix="renovate-automerge-shape-")
self.repo = pathlib.Path(self.temp.name)
subprocess.run(["git", "init", "-q"], cwd=self.repo, check=True)
git(self.repo, "config", "user.email", "test@example.com")
git(self.repo, "config", "user.name", "Test User")
source = self.repo / "apps" / "demo" / "1.0.0"
source.mkdir(parents=True)
(source / "docker-compose.yml").write_text(
"services:\n app:\n image: example/demo:1.0.0\n restart: unless-stopped\n",
encoding="utf-8",
)
(source / "data.yml").write_text("additionalProperties: {}\n", encoding="utf-8")
(source / ".env.sample").write_text("APP_DATA_DIR=./data\n", encoding="utf-8")
subprocess.run(["git", "add", "apps"], cwd=self.repo, check=True)
subprocess.run(["git", "commit", "-qm", "add source package"], cwd=self.repo, check=True)
self.base_sha = git(self.repo, "rev-parse", "HEAD")
def tearDown(self):
self.temp.cleanup()
def run_gate(self, files: list[dict[str, str]]) -> subprocess.CompletedProcess[str]:
head_sha = git(self.repo, "rev-parse", "HEAD")
files_path = self.repo / "pr-files.json"
output_path = self.repo / "shape.json"
files_path.write_text(json.dumps(files), encoding="utf-8")
return subprocess.run(
[
sys.executable,
str(SCRIPT),
"--base-sha",
self.base_sha,
"--head-sha",
head_sha,
"--files-json",
str(files_path),
"--output-json",
str(output_path),
],
cwd=self.repo,
text=True,
capture_output=True,
check=False,
)
def add_version_copy(self, *, modify_data: bool = False) -> list[dict[str, str]]:
source = self.repo / "apps" / "demo" / "1.0.0"
target = self.repo / "apps" / "demo" / "1.1.0"
shutil.copytree(source, target)
compose = target / "docker-compose.yml"
compose.write_text(
compose.read_text(encoding="utf-8").replace("example/demo:1.0.0", "example/demo:1.1.0"),
encoding="utf-8",
)
if modify_data:
(target / "data.yml").write_text("additionalProperties:\n unexpected: true\n", encoding="utf-8")
subprocess.run(["git", "add", "apps/demo/1.1.0"], cwd=self.repo, check=True)
subprocess.run(["git", "commit", "-qm", "add target version"], cwd=self.repo, check=True)
return [
{"filename": path.relative_to(self.repo).as_posix(), "status": "added"}
for path in sorted(target.rglob("*"))
if path.is_file()
]
def test_accepts_new_version_that_only_changes_image(self):
files = self.add_version_copy()
result = self.run_gate(files)
self.assertEqual(0, result.returncode, result.stderr)
payload = json.loads((self.repo / "shape.json").read_text(encoding="utf-8"))
self.assertEqual("added-version-package", payload["kind"])
self.assertFalse(payload["digest_only"])
def test_rejects_new_version_with_non_image_package_drift(self):
files = self.add_version_copy(modify_data=True)
result = self.run_gate(files)
self.assertEqual(42, result.returncode, result.stderr)
self.assertIn("does not exactly copy a base package", result.stderr)
def test_accepts_existing_single_service_image_only_update(self):
compose_path = self.repo / "apps" / "demo" / "1.0.0" / "docker-compose.yml"
compose_path.write_text(
compose_path.read_text(encoding="utf-8").replace("example/demo:1.0.0", "example/demo:1.1.0"),
encoding="utf-8",
)
subprocess.run(["git", "add", "apps/demo/1.0.0/docker-compose.yml"], cwd=self.repo, check=True)
subprocess.run(["git", "commit", "-qm", "update image"], cwd=self.repo, check=True)
result = self.run_gate(
[{"filename": "apps/demo/1.0.0/docker-compose.yml", "status": "modified"}]
)
self.assertEqual(0, result.returncode, result.stderr)
payload = json.loads((self.repo / "shape.json").read_text(encoding="utf-8"))
self.assertEqual("compose-update", payload["kind"])
self.assertFalse(payload["digest_only"])
def test_rejects_non_compose_file_in_existing_update(self):
compose_path = self.repo / "apps" / "demo" / "1.0.0" / "docker-compose.yml"
compose_path.write_text(
compose_path.read_text(encoding="utf-8").replace("example/demo:1.0.0", "example/demo:1.1.0"),
encoding="utf-8",
)
env_path = self.repo / "apps" / "demo" / "1.0.0" / ".env.sample"
env_path.write_text("APP_DATA_DIR=/unexpected\n", encoding="utf-8")
subprocess.run(["git", "add", "apps/demo/1.0.0"], cwd=self.repo, check=True)
subprocess.run(["git", "commit", "-qm", "change compose and env"], cwd=self.repo, check=True)
result = self.run_gate(
[
{"filename": "apps/demo/1.0.0/docker-compose.yml", "status": "modified"},
{"filename": "apps/demo/1.0.0/.env.sample", "status": "modified"},
]
)
self.assertEqual(42, result.returncode, result.stderr)
self.assertIn("non-Compose file", result.stderr)
if __name__ == "__main__":
unittest.main()