From 11034734257b7836e8bc14464f807aeb0732fffa Mon Sep 17 00:00:00 2001 From: WenyXu Date: Wed, 23 Sep 2026 08:39:23 +0000 Subject: [PATCH] ci: add optional AWS runners for observability benchmarks Signed-off-by: WenyXu --- .github/actions/setup-benchmark/action.yml | 38 ++ .../scripts/aliyun-ecs-runner-provision.py | 68 +-- .github/scripts/aliyun-ecs-runner-teardown.py | 16 +- .github/scripts/aws-ec2-runner-provision.py | 265 ++++++++++++ .github/scripts/aws-ec2-runner-teardown.py | 160 +++++++ .github/scripts/ci-runner-config.py | 47 ++ .github/scripts/ci-runners.md | 71 +++ .github/scripts/runner_utils.py | 101 +++++ .github/workflows/agent-observability.yml | 91 +++- .github/workflows/aws-ec2-runner-janitor.yml | 29 ++ .github/workflows/checks.yml | 1 + .github/workflows/vmbench-long-range.yml | 87 +++- tests/perf/test_aliyun_ecs_runner_scripts.py | 23 +- tests/perf/test_ci_runner_scripts.py | 406 ++++++++++++++++++ 14 files changed, 1288 insertions(+), 115 deletions(-) create mode 100644 .github/actions/setup-benchmark/action.yml create mode 100644 .github/scripts/aws-ec2-runner-provision.py create mode 100644 .github/scripts/aws-ec2-runner-teardown.py create mode 100644 .github/scripts/ci-runner-config.py create mode 100644 .github/scripts/ci-runners.md create mode 100644 .github/scripts/runner_utils.py create mode 100644 .github/workflows/aws-ec2-runner-janitor.yml create mode 100644 tests/perf/test_ci_runner_scripts.py diff --git a/.github/actions/setup-benchmark/action.yml b/.github/actions/setup-benchmark/action.yml new file mode 100644 index 00000000000..a446a78a2c3 --- /dev/null +++ b/.github/actions/setup-benchmark/action.yml @@ -0,0 +1,38 @@ +name: Set up benchmark dependencies +description: Reuse installed tools and install missing dependencies on an Ubuntu self-hosted runner +runs: + using: composite + steps: + - name: Check host and install missing tools + shell: bash + run: | + set -euo pipefail + [[ "$(uname -m)" == x86_64 ]] + command -v git + missing=() + for tool in curl jq python3 rsync; do + command -v "$tool" >/dev/null || missing+=("$tool") + done + if ((${#missing[@]})); then + sudo -n apt-get update + sudo -n env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends "${missing[@]}" + fi + - name: Install Docker if missing and verify access + shell: bash + run: | + set -euo pipefail + if ! command -v docker >/dev/null; then + . /etc/os-release + [[ "$ID" == ubuntu ]] + sudo -n install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -n tee /etc/apt/keyrings/docker.asc >/dev/null + sudo -n chmod a+r /etc/apt/keyrings/docker.asc + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" | sudo -n tee /etc/apt/sources.list.d/docker.list >/dev/null + sudo -n apt-get update + sudo -n env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin + fi + if ! docker info >/dev/null 2>&1; then + sudo -n systemctl enable --now docker + fi + # Membership must be configured before the runner starts, not in this job. + docker info diff --git a/.github/scripts/aliyun-ecs-runner-provision.py b/.github/scripts/aliyun-ecs-runner-provision.py index 1a26776b4cd..46efbffd3ec 100644 --- a/.github/scripts/aliyun-ecs-runner-provision.py +++ b/.github/scripts/aliyun-ecs-runner-provision.py @@ -49,10 +49,10 @@ import base64 import json import os import time -import urllib.error -import urllib.request from dataclasses import dataclass +from runner_utils import append_github_output, create_registration_token, find_runner_by_name, github_api + RUNNER_NAME_PREFIX = "qreg-ecs" RUNNER_LABEL_PREFIX = "query-regression-ecs" MANAGED_BY_TAG_KEY = "managed-by" @@ -243,63 +243,6 @@ def encode_user_data(script: str) -> str: return base64.b64encode(script.encode("utf-8")).decode("ascii") -def github_api(token: str, method: str, path: str, body: dict | None = None) -> dict: - data = json.dumps(body).encode("utf-8") if body is not None else None - request = urllib.request.Request( - f"https://api.github.com{path}", - data=data, - method=method, - headers={ - "Accept": "application/vnd.github+json", - "Authorization": f"Bearer {token}", - "X-GitHub-Api-Version": "2022-11-28", - }, - ) - try: - with urllib.request.urlopen(request, timeout=30) as response: - if response.status == 204: - return {} - return json.loads(response.read().decode("utf-8")) - except urllib.error.HTTPError as error: - # GitHub's error body says exactly why (e.g. "Must have admin rights to - # Repository" for a PAT without the required scope); surface it instead - # of a bare "HTTP Error 403". - body = error.read().decode("utf-8", "replace") - raise SystemExit( - f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n" - "The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs " - "'repo' scope (classic PAT) or 'Administration: write' on the " - "repository (fine-grained PAT)." - ) from error - - -def create_registration_token(github_token: str, repo: str) -> str: - response = github_api( - github_token, - "POST", - f"/repos/{repo}/actions/runners/registration-token", - body={}, - ) - return response["token"] - - -def find_runner_by_name(github_token: str, repo: str, name: str) -> dict | None: - page = 1 - while True: - response = github_api( - github_token, - "GET", - f"/repos/{repo}/actions/runners?per_page=100&page={page}", - ) - runners = response.get("runners", []) - for runner in runners: - if runner.get("name") == name: - return runner - if len(runners) < 100: - return None - page += 1 - - def make_ecs_client(config: ProvisionConfig): from alibabacloud_ecs20140526.client import Client as EcsClient from alibabacloud_tea_openapi.models import Config as OpenApiConfig @@ -388,13 +331,6 @@ def dump_console_output(client, region_id: str, instance_id: str) -> None: summary.write("\n```\n\n") -def append_github_output(name: str, value: str) -> None: - output_path = os.environ.get("GITHUB_OUTPUT") - if output_path: - with open(output_path, "a", encoding="utf-8") as output: - output.write(f"{name}={value}\n") - - def run_instance(client, config: ProvisionConfig, user_data: str) -> str: from alibabacloud_ecs20140526 import models as ecs_models diff --git a/.github/scripts/aliyun-ecs-runner-teardown.py b/.github/scripts/aliyun-ecs-runner-teardown.py index 658633e0395..d5b30b2922d 100644 --- a/.github/scripts/aliyun-ecs-runner-teardown.py +++ b/.github/scripts/aliyun-ecs-runner-teardown.py @@ -47,6 +47,8 @@ import sys from datetime import datetime, timedelta, timezone from pathlib import Path +from runner_utils import deregister_runner + # Reuse the GitHub API client, runner lookup, and tag constants from the # provision script. _PROVISION_SPEC = importlib.util.spec_from_file_location( @@ -159,20 +161,6 @@ def delete_instance(client, instance_id: str, region_id: str | None = None) -> b return False -def deregister_runner(token: str, repo: str, runner_name: str) -> bool: - runner = provision.find_runner_by_name(token, repo, runner_name) - if runner is None: - print(f"Runner {runner_name} is not registered", flush=True) - return True - try: - provision.github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}") - print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True) - return True - except Exception as error: # noqa: BLE001 - print(f"Failed to deregister runner {runner_name}: {error}", flush=True) - return False - - def list_managed_instances( client, region_id: str ) -> list[tuple[str, str, str, str | None]]: diff --git a/.github/scripts/aws-ec2-runner-provision.py b/.github/scripts/aws-ec2-runner-provision.py new file mode 100644 index 00000000000..7bf1b4091c9 --- /dev/null +++ b/.github/scripts/aws-ec2-runner-provision.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# /// script +# requires-python = ">=3.10" +# dependencies = ["boto3>=1.35,<2"] +# /// + +"""Provision an ephemeral EC2 CI runner from an Ubuntu x86_64 AMI.""" + +from __future__ import annotations + +import argparse +import os +import re +import shlex +import time +from dataclasses import dataclass + +from runner_utils import append_github_output, create_registration_token, find_runner_by_name, github_api + +MANAGED_BY = "query-regression-ci" +REPO_TAG = "github-repository" +RUN_TAG = "query-regression-run-id" +RUNNER_TAG = "github-runner-name" +TTL_TAG_KEY = "runner-ttl-hours" + + +def make_client(region: str): + import boto3 + from botocore.config import Config + + return boto3.client("ec2", region_name=region, config=Config( + connect_timeout=10, read_timeout=30, + retries={"mode": "standard", "max_attempts": 3}, + )) + + +@dataclass(frozen=True) +class ProvisionConfig: + region: str + image_id: str + instance_type: str + subnet_id: str + security_group_id: str + repo: str + run_id: str + github_token: str + system_disk_gib: int = 500 + ttl_hours: int = 8 + + def __post_init__(self): + if not re.fullmatch(r"[0-9]+-[0-9]+", self.run_id): + raise ValueError("run-id must be GitHub run_id-run_attempt") + if not 20 <= self.system_disk_gib <= 2048: + raise ValueError("system-disk-gib must be between 20 and 2048") + if not 1 <= self.ttl_hours <= 168: + raise ValueError("ttl-hours must be between 1 and 168") + + @property + def runner_name(self) -> str: + return f"ci-ec2-{self.run_id}" + + +def resolve_image(client, image_id: str) -> str: + if image_id != "auto": + return image_id + images = client.describe_images( + Owners=["099720109477"], + Filters=[ + {"Name": "name", "Values": ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]}, + {"Name": "state", "Values": ["available"]}, + {"Name": "architecture", "Values": ["x86_64"]}, + ], + )["Images"] + if not images: + raise ValueError("No Canonical Ubuntu 24.04 amd64 AMI found in this region") + image = max(images, key=lambda image: image["CreationDate"]) + print(f"Using AMI {image['ImageId']}: {image['Name']}") + return image["ImageId"] + + +def root_disk(client, image_id: str, size: int) -> dict: + images = client.describe_images(ImageIds=[image_id])["Images"] + if len(images) != 1: + raise ValueError(f"AMI not found: {image_id}") + image = images[0] + if image["State"] != "available" or image["Architecture"] != "x86_64": + raise ValueError("Runner AMI must be available and x86_64") + root = next((d for d in image["BlockDeviceMappings"] + if d["DeviceName"] == image["RootDeviceName"] and "Ebs" in d), None) + if root is None: + raise ValueError("Runner AMI must have an EBS root disk") + minimum = root["Ebs"]["VolumeSize"] + if size < minimum: + raise ValueError(f"Requested root disk {size} GiB is smaller than AMI minimum {minimum} GiB") + return {"DeviceName": image["RootDeviceName"], "Ebs": { + "VolumeSize": size, "VolumeType": "gp3", "Iops": 3000, + "Throughput": 125, "DeleteOnTermination": True, + }} + + +def runner_download(token: str, repo: str) -> str: + runners = github_api(token, "GET", f"/repos/{repo}/actions/runners/downloads") + return next(r["download_url"] for r in runners if r["os"] == "linux" and r["architecture"] == "x64") + + +def render_user_data(name: str, token: str, repo: str, download_url: str) -> str: + # Quote every external value before embedding it in the root shell script. + name, token, repo, download_url = map(shlex.quote, (name, token, repo, download_url)) + return f"""#!/bin/bash +set -euo pipefail +for tool in git curl tar sudo; do command -v "$tool"; done +# Mask systemd-oomd before enabling swap: Ubuntu 24.04 kills the whole +# service cgroup on PSI pressure, and swap thrashing looks like pressure. +systemctl disable --now systemd-oomd.socket systemd-oomd.service || true +systemctl mask systemd-oomd.socket systemd-oomd.service || true +# apt-daily-upgrade / unattended-upgrades can `systemctl restart` services +# whose libraries were updated. That SIGTERM-s the runner mid-job; GitHub +# records UserCancelled (job still valid, child exit 143). These VMs live +# ~1h; security updates belong in the image, not at job time. +systemctl disable --now unattended-upgrades.service apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service || true +systemctl mask unattended-upgrades.service apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service || true +systemctl stop unattended-upgrades.service || true +killall -9 unattended-upgr || true +cat > /etc/apt/apt.conf.d/99disable-auto-updates <<'APTEOF' +APT::Periodic::Update-Package-Lists "0"; +APT::Periodic::Unattended-Upgrade "0"; +APT::Periodic::Download-Upgradeable-Packages "0"; +APTEOF +if [[ ! -f "/swapfile" ]]; then + fallocate --length 16G "/swapfile" + chmod 600 "/swapfile" + mkswap "/swapfile" +fi +swapon "/swapfile" +sysctl --write vm.swappiness=10 +swapon --show +free --human + +id runner >/dev/null 2>&1 || useradd --create-home --shell /bin/bash runner +# Establish membership before the runner service starts; setup installs Docker later. +getent group docker >/dev/null || groupadd --system docker +usermod -aG docker runner +printf 'runner ALL=(ALL) NOPASSWD: ALL\\n' > /etc/sudoers.d/ci-runner +chmod 0440 /etc/sudoers.d/ci-runner +visudo -cf /etc/sudoers.d/ci-runner +install -d -o runner -g "$(id -gn runner)" /home/runner/benchmark-data /opt/ci-github-runner +cd /opt/ci-github-runner +curl --fail --location --retry 3 {download_url} | tar xz +./bin/installdependencies.sh +chown -R runner:"$(id -gn runner)" /opt/ci-github-runner +runuser -u runner -- ./config.sh --unattended --ephemeral --disableupdate --url https://github.com/{repo} --token {token} --name {name} --labels {name} +./svc.sh install runner +# Match the ECS runner's behavior when a child process is OOM-killed. +service_name=$(cat .service) +mkdir -p "/etc/systemd/system/$service_name.d" +cat > "/etc/systemd/system/$service_name.d/oom.conf" <<'EOF' +[Service] +OOMPolicy=continue +EOF +systemctl daemon-reload +# Package installation in a job must not restart its runner service. +install -d /etc/needrestart/conf.d +cat > /etc/needrestart/conf.d/actions_runner_services.conf <<'EOF' +$nrconf{{override_rc}}{{qr(^actions\\.runner\\..+\\.service$)}} = 0; +EOF +./svc.sh start +""" + + +def run_instance(client, config: ProvisionConfig) -> str: + image_id = resolve_image(client, config.image_id) + disk = root_disk(client, image_id, config.system_disk_gib) + instance = client.describe_instance_types(InstanceTypes=[config.instance_type])["InstanceTypes"][0] + if "x86_64" not in instance["ProcessorInfo"]["SupportedArchitectures"]: + raise ValueError("Runner instance type must support x86_64") + token = create_registration_token(config.github_token, config.repo) + print(f"::add-mask::{token}") + script = render_user_data(config.runner_name, token, config.repo, runner_download(config.github_token, config.repo)) + tags = [ + {"Key": "Name", "Value": config.runner_name}, + {"Key": "managed-by", "Value": MANAGED_BY}, + {"Key": REPO_TAG, "Value": config.repo}, + {"Key": RUN_TAG, "Value": config.run_id}, + {"Key": RUNNER_TAG, "Value": config.runner_name}, + {"Key": TTL_TAG_KEY, "Value": str(config.ttl_hours)}, + ] + response = client.run_instances( + ImageId=image_id, InstanceType=config.instance_type, + MinCount=1, MaxCount=1, SubnetId=config.subnet_id, + SecurityGroupIds=[config.security_group_id], + BlockDeviceMappings=[disk], UserData=script, + ClientToken=config.runner_name, + MetadataOptions={"HttpTokens": "required", "HttpEndpoint": "enabled"}, + TagSpecifications=[{"ResourceType": kind, "Tags": tags} for kind in ("instance", "volume")], + ) + return response["Instances"][0]["InstanceId"] + + +def dump_console(client, instance_id: str) -> None: + try: + # Boto3 decodes the API's base64 console output. + output = client.get_console_output(InstanceId=instance_id, Latest=True).get("Output", "") + print(f"::group::EC2 console {instance_id}") + print("\n".join(output.splitlines()[-80:])) + print("::endgroup::") + except Exception as error: + print(f"Unable to read console for {instance_id}: {error}") + + +def provision(client, config: ProvisionConfig) -> int: + instance_id = run_instance(client, config) + # Publish before waiting: failed registration must still reach teardown. + for key, value in (("instance_id", instance_id), + ("label", config.runner_name), ("runner_name", config.runner_name)): + append_github_output(key, value) + try: + client.get_waiter("instance_running").wait( + InstanceIds=[instance_id], WaiterConfig={"Delay": 5, "MaxAttempts": 60}) + deadline = time.monotonic() + 10 * 60 + while time.monotonic() < deadline: + runner = find_runner_by_name(config.github_token, config.repo, config.runner_name) + if runner and runner.get("status") == "online": + print(f"Runner {config.runner_name} online: {instance_id}") + return 0 + time.sleep(5) + raise TimeoutError(f"Runner {config.runner_name} did not come online") + finally: + dump_console(client, instance_id) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + for name, env in (("region", "AWS_REGION"), ("repo", "GITHUB_REPOSITORY"), + ("github-token", "GH_PERSONAL_ACCESS_TOKEN"), + ("run-id", "AWS_EC2_RUN_ID"), ("image-id", "AWS_EC2_IMAGE_ID"), + ("instance-type", "AWS_EC2_INSTANCE_TYPE"), + ("subnet-id", "AWS_EC2_SUBNET_ID"), + ("security-group-id", "AWS_EC2_SECURITY_GROUP_ID")): + parser.add_argument(f"--{name}", default=os.environ.get(env) or ("auto" if name == "image-id" else None)) + parser.add_argument("--system-disk-gib", type=int, default=os.environ.get("AWS_EC2_SYSTEM_DISK_GIB", "500")) + parser.add_argument("--ttl-hours", type=int, default=os.environ.get("AWS_EC2_TTL_HOURS", "8")) + args = parser.parse_args() + for name, value in vars(args).items(): + if value is None or value == "": + parser.error(f"Missing {name.replace('_', '-')}") + config = ProvisionConfig(**vars(args)) + return provision(make_client(args.region), config) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/aws-ec2-runner-teardown.py b/.github/scripts/aws-ec2-runner-teardown.py new file mode 100644 index 00000000000..fbfba4f639a --- /dev/null +++ b/.github/scripts/aws-ec2-runner-teardown.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# /// script +# requires-python = ">=3.10" +# dependencies = ["boto3>=1.35,<2"] +# /// + +"""Delete managed EC2 CI runners, or sweep expired instances by ownership tags.""" + +from __future__ import annotations + +import argparse +import importlib.util +import os +import sys +import time +from pathlib import Path +from datetime import datetime, timedelta, timezone + +from runner_utils import deregister_runner + +_SPEC = importlib.util.spec_from_file_location("aws_ec2_runner_provision", Path(__file__).with_name("aws-ec2-runner-provision.py")) +assert _SPEC is not None and _SPEC.loader is not None +provision = importlib.util.module_from_spec(_SPEC) +sys.modules[_SPEC.name] = provision +_SPEC.loader.exec_module(provision) +make_client = provision.make_client +dump_console = provision.dump_console +MANAGED_BY = provision.MANAGED_BY +REPO_TAG = provision.REPO_TAG +RUN_TAG = provision.RUN_TAG +RUNNER_TAG = provision.RUNNER_TAG +TTL_TAG_KEY = provision.TTL_TAG_KEY +SWEEP_TTL_HOURS = 4 + + +def managed_instances(client, repo: str, run_id: str | None = None, + instance_id: str | None = None) -> list[dict]: + filters = [{"Name": "tag:managed-by", "Values": [MANAGED_BY]}, + {"Name": f"tag:{REPO_TAG}", "Values": [repo]}] + if instance_id: + # A retried teardown may have a newer workflow attempt than creation. + filters.append({"Name": "instance-id", "Values": [instance_id]}) + elif run_id: + filters.append({"Name": f"tag:{RUN_TAG}", "Values": [run_id]}) + return [instance + for page in client.get_paginator("describe_instances").paginate(Filters=filters) + for reservation in page["Reservations"] for instance in reservation["Instances"]] + + +def delete_instance(client, instance_id: str) -> bool: + dump_console(client, instance_id) + for attempt in range(12): + try: + client.terminate_instances(InstanceIds=[instance_id]) + client.get_waiter("instance_terminated").wait( + InstanceIds=[instance_id], WaiterConfig={"Delay": 5, "MaxAttempts": 60}) + return True + except Exception as error: + code = getattr(error, "response", {}).get("Error", {}).get("Code") + if code == "InvalidInstanceID.NotFound": + if attempt < 11: + time.sleep(5) + continue + # The ownership lookup observed this instance. Treat sustained + # NotFound after bounded retries as an already-deleted VM. + return True + print(f"Failed to terminate {instance_id}: {error}") + return False + return False + + +def teardown(client, repo: str, token: str, run_id: str | None = None, + instance_id: str | None = None, now: datetime | None = None, + runner_name: str | None = None) -> int: + # Lookup by ownership/run tags also recovers a create whose response or job + # outputs were lost. Sweep never touches legacy build/JSONBench instances. + instances = managed_instances(client, repo, run_id, instance_id) + # Tags can lag RunInstances. Retry before concluding a targeted VM is gone. + if run_id and not instances: + for _ in range(11): + time.sleep(5) + instances = managed_instances(client, repo, run_id, instance_id) + if instances: + break + ok = True + found = False + for instance in instances: + identity = instance["InstanceId"] + if instance_id and identity != instance_id: + continue + found = True + tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])} + name = tags.get(RUNNER_TAG, "") + if not name.startswith("ci-ec2-"): + print(f"Skipping {identity}: missing managed runner name") + ok = False + continue + if not run_id: + ttl = tags.get(TTL_TAG_KEY) + try: + hours = SWEEP_TTL_HOURS if ttl is None else int(ttl) + if not 1 <= hours <= 168: + raise ValueError("TTL outside 1..168 hours") + except (TypeError, ValueError): + print(f"Skipping {identity}: invalid TTL {ttl!r}") + continue + if (now or datetime.now(timezone.utc)) - instance["LaunchTime"] < timedelta(hours=hours): + continue + # A GitHub outage must not prevent EC2 deletion, or vice versa. + ok &= delete_instance(client, identity) + try: + ok &= deregister_runner(token, repo, name) + except (Exception, SystemExit) as error: + print(f"Failed to unregister {name}: {error}") + ok = False + if run_id and not found: + # EC2 may already be gone while its GitHub registration remains. + name = runner_name or f"ci-ec2-{run_id}" + if not name.startswith("ci-ec2-"): + raise ValueError("Invalid CI runner name") + ok &= deregister_runner(token, repo, name) + return 0 if ok else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + for name, env in (("region", "AWS_REGION"), ("repo", "GITHUB_REPOSITORY"), + ("github-token", "GH_PERSONAL_ACCESS_TOKEN"), + ("run-id", "AWS_EC2_RUN_ID"), ("instance-id", "AWS_EC2_INSTANCE_ID"), + ("runner-name", "AWS_EC2_RUNNER_NAME")): + parser.add_argument(f"--{name}", default=os.environ.get(env)) + parser.add_argument("--sweep", action="store_true") + args = parser.parse_args() + for name in ("region", "repo", "github_token"): + if not getattr(args, name): + parser.error(f"Missing {name.replace('_', '-')}") + if not args.sweep and not args.run_id: + parser.error("Missing run-id for targeted teardown") + return teardown(make_client(args.region), args.repo, args.github_token, + None if args.sweep else args.run_id, + None if args.sweep else args.instance_id, + runner_name=None if args.sweep else args.runner_name) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/ci-runner-config.py b/.github/scripts/ci-runner-config.py new file mode 100644 index 00000000000..a242b263c3e --- /dev/null +++ b/.github/scripts/ci-runner-config.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve dispatch provider and instance type before allocating a runner.""" + +import argparse +import re + +from runner_utils import append_github_output + +DEFAULT_INSTANCE_TYPES = {"Aliyun": "ecs.c9i.2xlarge", "AWS": "c7i.2xlarge"} + + +def resolve_instance_type(provider: str, requested: str) -> str: + if provider not in DEFAULT_INSTANCE_TYPES: + raise ValueError("provider must be Aliyun or AWS") + instance_type = DEFAULT_INSTANCE_TYPES[provider] if requested == "auto" else requested + pattern = r"ecs\.[a-z0-9-]+\.[a-z0-9-]+" if provider == "Aliyun" else r"[a-z0-9-]+\.[a-z0-9-]+" + if not re.fullmatch(pattern, instance_type): + raise ValueError(f"Invalid {provider} instance type: {instance_type!r}") + return instance_type + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--provider", required=True) + parser.add_argument("--instance-type", default="auto") + args = parser.parse_args() + instance_type = resolve_instance_type(args.provider, args.instance_type) + append_github_output("instance_type", instance_type) + print(f"CI runner: {args.provider} / {instance_type}") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/ci-runners.md b/.github/scripts/ci-runners.md new file mode 100644 index 00000000000..8215c208d2a --- /dev/null +++ b/.github/scripts/ci-runners.md @@ -0,0 +1,71 @@ +# CI runners + +The agent observability and long-range benchmark workflows select `Aliyun` or +`AWS` through the `provider` dispatch input. `instance_type: auto` selects +`ecs.c9i.2xlarge` or `c7i.2xlarge`, respectively (8 vCPU, 16 GiB). +Existing build/release and JSONBench workflows keep their current runner actions. + +Aliyun uses the existing ECS actions and image. Its provision and teardown +scripts share only GitHub API and output helpers from `runner_utils.py` with +the new AWS scripts. + +AWS uses `aws-ec2-runner-provision.py` and `aws-ec2-runner-teardown.py` directly. +Repository variables supply `EC2_RUNNER_REGION`, `EC2_RUNNER_SUBNET_ID`, and +`EC2_RUNNER_SECURITY_GROUP_ID`; credentials use `AWS_ACCESS_KEY_ID`, +`AWS_SECRET_ACCESS_KEY`, and `GH_PERSONAL_ACCESS_TOKEN` secrets. +The current region is Singapore (`ap-southeast-1`). By default the script selects +the newest available Canonical Ubuntu 24.04 amd64 server AMI in that region +(owner `099720109477`) and logs its ID. Set `BENCHMARK_EC2_IMAGE_ID` to pin a +prepared Ubuntu AMI. Existing build CI's `EC2_RUNNER_LINUX_AMD64_IMAGE_ID` is +unaffected. Both benchmark workflows default to 500 GiB for both providers; +explicit disk overrides must be at least the selected AMI's root snapshot size. +The official 24.04 image does not have the old custom AMI's 500 GiB minimum. + +AWS user data registers a fresh ephemeral runner. Git, curl, tar, and sudo are +expected in the Ubuntu server image. Docker group membership is set before the +runner service starts, and this dedicated VM grants the runner passwordless sudo +so its job can install dependencies. Like ECS, the host creates 16 GiB swap with +swappiness 10, disables systemd-oomd and background automatic updates, and sets +`OOMPolicy=continue` on the runner service. Explicit setup package installation +remains enabled. + +Both benchmark jobs call `.github/actions/setup-benchmark` after checkout. It +reuses existing tools, installs missing curl/jq/python3/rsync, installs Docker CE +if absent, and checks Docker access. Missing packages require passwordless sudo; +Aliyun's prepared image normally provides the tools and Docker already. Existing +Docker installations are preserved, so exact versions need not match between +providers. Provisioning must establish Docker permissions before starting the +runner; adding a group inside the running job would not update its groups. + +The subnet must provide outbound access to Ubuntu and Docker packages, GitHub, +and the configured container registries. The IAM identity needs EC2 +image/instance-type discovery, instance and volume tagging, launch, describe, +console output, and termination permissions. GitHub credentials need repository +runner administration access. + +Creation publishes instance identity before waiting for registration. Teardown +can recover missing outputs using repository and workflow-attempt tags. The +daily AWS janitor (03:23 UTC) removes expired instances tagged `managed-by=query-regression-ci` +and unregisters their runners. Both providers use the existing +`managed-by=query-regression-ci`, `query-regression-run-id`, and +`runner-ttl-hours` tags. AWS retains a run-attempt suffix in its run identity to +isolate retries, plus repository and runner-name tags to restrict cleanup. +Legacy build instances do not match these ownership filters. +Both janitors use a 4-hour fallback when the TTL tag is absent and skip invalid +TTL tags. Benchmark jobs explicitly tag their configured TTL (default 8 hours). +Cleanup does not check whether the GitHub job is still running. Expired instances +are removed on the next daily sweep; normal workflow teardown runs immediately. +TTL is measured from instance launch, so it must cover provisioning and the +benchmark. The scheduled janitor becomes active after landing on the default +branch. + +Local validation: + +```sh +python3 tests/perf/test_ci_runner_scripts.py +python3 tests/perf/test_aliyun_ecs_runner_scripts.py +actionlint .github/workflows/agent-observability.yml .github/workflows/vmbench-long-range.yml .github/workflows/aws-ec2-runner-janitor.yml +``` + +A real AWS dispatch is still needed to verify AMI boot, package +installation, registry connectivity, and end-to-end benchmark execution. diff --git a/.github/scripts/runner_utils.py b/.github/scripts/runner_utils.py new file mode 100644 index 00000000000..afd827f62cf --- /dev/null +++ b/.github/scripts/runner_utils.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Small shared GitHub runner API and workflow-output helpers.""" + +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request + + +def github_api(token: str, method: str, path: str, body: dict | None = None) -> dict: + data = json.dumps(body).encode("utf-8") if body is not None else None + request = urllib.request.Request( + f"https://api.github.com{path}", + data=data, + method=method, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + if response.status == 204: + return {} + return json.loads(response.read().decode("utf-8")) + except urllib.error.HTTPError as error: + # GitHub's error body says exactly why (e.g. "Must have admin rights to + # Repository" for a PAT without the required scope); surface it instead + # of a bare "HTTP Error 403". + body = error.read().decode("utf-8", "replace") + raise SystemExit( + f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n" + "The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs " + "'repo' scope (classic PAT) or 'Administration: write' on the " + "repository (fine-grained PAT)." + ) from error + + +def create_registration_token(github_token: str, repo: str) -> str: + response = github_api( + github_token, + "POST", + f"/repos/{repo}/actions/runners/registration-token", + body={}, + ) + return response["token"] + + +def find_runner_by_name(github_token: str, repo: str, name: str) -> dict | None: + page = 1 + while True: + response = github_api( + github_token, + "GET", + f"/repos/{repo}/actions/runners?per_page=100&page={page}", + ) + runners = response.get("runners", []) + for runner in runners: + if runner.get("name") == name: + return runner + if len(runners) < 100: + return None + page += 1 + + +def append_github_output(name: str, value: str) -> None: + output_path = os.environ.get("GITHUB_OUTPUT") + if output_path: + with open(output_path, "a", encoding="utf-8") as output: + output.write(f"{name}={value}\n") + + +def deregister_runner(token: str, repo: str, runner_name: str) -> bool: + runner = find_runner_by_name(token, repo, runner_name) + if runner is None: + print(f"Runner {runner_name} is not registered", flush=True) + return True + try: + github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}") + print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True) + return True + except Exception as error: # noqa: BLE001 + print(f"Failed to deregister runner {runner_name}: {error}", flush=True) + return False diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index 02c4ccb81bc..38f644b78df 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -50,15 +50,21 @@ on: type: string default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0 required: true - ecs_instance_type: - description: ECS instance type (independent of Query Regression) + provider: + description: Cloud provider for the benchmark runner + type: choice + options: [Aliyun, AWS] + default: Aliyun + required: true + instance_type: + description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.' type: string - default: ecs.c9i.2xlarge + default: auto required: true system_disk_gib: - description: System disk GiB; 0 = auto (S 80, P 100, M 500), or 20..2048 + description: System disk GiB (default 500; 0 uses default) type: number - default: 0 + default: 500 required: true benchmark_timeout_minutes: description: Benchmark job timeout in minutes (1..360) @@ -89,7 +95,8 @@ env: VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }} O11YBENCH_REF: ${{ inputs.o11ybench_ref }} RUNTIME_IMAGE: ${{ inputs.runtime_image }} - ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }} + PROVIDER: ${{ inputs.provider }} + REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }} JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} @@ -100,6 +107,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: + instance_type: ${{ steps.runner.outputs.instance_type }} targets: ${{ steps.inputs.outputs.targets }} system_disk_gib: ${{ steps.inputs.outputs.system_disk_gib }} greptimedb: ${{ steps.inputs.outputs.greptimedb }} @@ -110,9 +118,13 @@ jobs: with: path: greptimedb persist-credentials: false - - name: Test ECS lifecycle helpers + - name: Resolve CI runner + id: runner + run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" + - name: Test CI runner helpers run: | python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py + python3 greptimedb/tests/perf/test_ci_runner_scripts.py python3 greptimedb/tests/perf/test_agent_observability_summary.py - name: Validate requested benchmark id: inputs @@ -124,14 +136,11 @@ jobs: run: | set -euo pipefail # Validate cloud lifecycle budgets before provisioning; runtime owns workload validation. - [[ -n "$ECS_INSTANCE_TYPE" ]] case "$PROFILE" in - S) auto_disk_gib=80 ;; - P) auto_disk_gib=100 ;; - M) auto_disk_gib=500 ;; + S|P|M) ;; *) echo "Unknown dataset: $PROFILE" >&2; exit 1 ;; esac - [[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=$auto_disk_gib + [[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=500 for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do [[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; } done @@ -160,15 +169,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 outputs: - label: ${{ steps.ecs.outputs.label }} - instance_id: ${{ steps.ecs.outputs.instance_id }} - runner_name: ${{ steps.ecs.outputs.runner_name }} + label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }} + instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }} + runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }} steps: - uses: actions/checkout@v4 with: path: greptimedb persist-credentials: false - name: Create ephemeral ECS runner + if: inputs.provider == 'Aliyun' id: ecs uses: ./greptimedb/.github/actions/aliyun-ecs-create with: @@ -179,13 +189,35 @@ jobs: vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }} security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }} image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} - instance-type: ${{ env.ECS_INSTANCE_TYPE }} + instance-type: ${{ needs.validate.outputs.instance_type }} enable-docker: 'true' system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }} ttl-hours: ${{ inputs.janitor_ttl_hours }} resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} + - uses: aws-actions/configure-aws-credentials@v4 + if: inputs.provider == 'AWS' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + if: inputs.provider == 'AWS' + - name: Create ephemeral EC2 runner + if: inputs.provider == 'AWS' + id: aws + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }} + AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }} + AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }} + AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }} + AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py benchmark: needs: - validate @@ -193,6 +225,7 @@ jobs: runs-on: ${{ needs.provision.outputs.label }} timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }} env: + INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }} TARGETS: ${{ needs.validate.outputs.targets }} SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }} OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }} @@ -209,6 +242,8 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Set up benchmark dependencies + uses: ./greptimedb/.github/actions/setup-benchmark - name: Check Docker on ephemeral Linux host shell: bash run: | @@ -252,9 +287,9 @@ jobs: done jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \ --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \ - --arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \ + --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \ --argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \ - '{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH" + '{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ($provider | if . == "AWS" then "ec2" else "ecs" end):{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH" - name: Checkout benchmark tools uses: actions/checkout@v4 with: @@ -343,7 +378,7 @@ jobs: needs: - provision - benchmark - if: ${{ always() && needs.provision.outputs.instance_id != '' }} + if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -352,6 +387,7 @@ jobs: path: greptimedb persist-credentials: false - name: Delete ECS and unregister runner + if: inputs.provider == 'Aliyun' uses: ./greptimedb/.github/actions/aliyun-ecs-delete with: access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} @@ -360,3 +396,20 @@ jobs: region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} instance-id: ${{ needs.provision.outputs.instance_id }} runner-name: ${{ needs.provision.outputs.runner_name }} + - uses: aws-actions/configure-aws-credentials@v4 + if: inputs.provider == 'AWS' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + if: inputs.provider == 'AWS' + - name: Delete EC2 and unregister runner + if: inputs.provider == 'AWS' + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }} + AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py diff --git a/.github/workflows/aws-ec2-runner-janitor.yml b/.github/workflows/aws-ec2-runner-janitor.yml new file mode 100644 index 00000000000..1c4435022b4 --- /dev/null +++ b/.github/workflows/aws-ec2-runner-janitor.yml @@ -0,0 +1,29 @@ +name: AWS EC2 Runner Janitor + +on: + schedule: + - cron: '23 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +jobs: + sweep: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + - name: Sweep expired CI instances and unregister runners + env: + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + run: uv run .github/scripts/aws-ec2-runner-teardown.py --sweep diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6e591ba9dfd..db426da86a9 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -97,6 +97,7 @@ jobs: python3 tests/perf/test_query_regression_nightly_refs.py python3 tests/perf/test_query_regression_slash.py python3 tests/perf/test_aliyun_ecs_runner_scripts.py + python3 tests/perf/test_ci_runner_scripts.py python3 tests/perf/test_agent_observability_summary.py check: diff --git a/.github/workflows/vmbench-long-range.yml b/.github/workflows/vmbench-long-range.yml index 9d0088217ad..d700400de98 100644 --- a/.github/workflows/vmbench-long-range.yml +++ b/.github/workflows/vmbench-long-range.yml @@ -35,10 +35,16 @@ on: type: string default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29 required: true - ecs_instance_type: - description: ECS instance type (independent of Query Regression) + provider: + description: Cloud provider for the benchmark runner + type: choice + options: [Aliyun, AWS] + default: Aliyun + required: true + instance_type: + description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.' type: string - default: ecs.c9i.2xlarge + default: auto required: true benchmark_root: description: Absolute disk-backed work directory (outside the o11ybench checkout) @@ -46,9 +52,9 @@ on: default: /home/runner/benchmark-data required: true system_disk_gib: - description: System disk GiB (20..2048) + description: System disk GiB (20..2048, default 500) type: number - default: 80 + default: 500 required: true benchmark_timeout_minutes: description: Benchmark job timeout in minutes (1..360) @@ -78,7 +84,8 @@ env: GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }} RUNTIME_IMAGE: ${{ inputs.runtime_image }} - ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }} + PROVIDER: ${{ inputs.provider }} + REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }} JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} @@ -89,6 +96,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 outputs: + instance_type: ${{ steps.runner.outputs.instance_type }} system_disk_gib: ${{ steps.plan.outputs.system_disk_gib }} runtime_image: ${{ steps.plan.outputs.runtime_image }} o11ybench_ref: ${{ steps.plan.outputs.o11ybench_ref }} @@ -99,8 +107,13 @@ jobs: with: path: greptimedb persist-credentials: false - - name: Test existing ECS lifecycle helpers - run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py + - name: Resolve CI runner + id: runner + run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" + - name: Test CI runner helpers + run: | + python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py + python3 greptimedb/tests/perf/test_ci_runner_scripts.py - name: Validate budgets and build warm plan id: plan env: @@ -108,7 +121,6 @@ jobs: RUN_VICTORIAMETRICS: ${{ inputs.run_victoriametrics }} run: | set -euo pipefail - [[ -n "$ECS_INSTANCE_TYPE" ]] [[ "$BENCHMARK_ROOT" == /* && "$BENCHMARK_ROOT" != / && "$BENCHMARK_ROOT" != */ ]] [[ "$BENCHMARK_ROOT" != /tmp && "$BENCHMARK_ROOT" != /tmp/* ]] [[ "$BENCHMARK_ROOT" != *$'\n'* && "$BENCHMARK_ROOT" != *$'\r'* ]] @@ -159,15 +171,16 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 outputs: - label: ${{ steps.ecs.outputs.label }} - instance_id: ${{ steps.ecs.outputs.instance_id }} - runner_name: ${{ steps.ecs.outputs.runner_name }} + label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }} + instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }} + runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }} steps: - uses: actions/checkout@v4 with: path: greptimedb persist-credentials: false - name: Create ephemeral ECS runner + if: inputs.provider == 'Aliyun' id: ecs uses: ./greptimedb/.github/actions/aliyun-ecs-create with: @@ -178,13 +191,35 @@ jobs: vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }} security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }} image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} - instance-type: ${{ env.ECS_INSTANCE_TYPE }} + instance-type: ${{ needs.validate.outputs.instance_type }} enable-docker: 'true' system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }} ttl-hours: ${{ inputs.janitor_ttl_hours }} resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} + - uses: aws-actions/configure-aws-credentials@v4 + if: inputs.provider == 'AWS' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + if: inputs.provider == 'AWS' + - name: Create ephemeral EC2 runner + if: inputs.provider == 'AWS' + id: aws + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }} + AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }} + AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }} + AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }} + AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py benchmark: needs: - validate @@ -202,6 +237,12 @@ jobs: TSDG_BATCH_SIZE: '100000' O11YBENCH_GENERATOR_CPUS: '4' steps: + - uses: actions/checkout@v4 + with: + path: greptimedb + persist-credentials: false + - name: Set up benchmark dependencies + uses: ./greptimedb/.github/actions/setup-benchmark - name: Checkout matching benchmark tools uses: actions/checkout@v4 with: @@ -283,7 +324,7 @@ jobs: needs: - provision - benchmark - if: ${{ always() && needs.provision.outputs.instance_id != '' }} + if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }} runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -292,6 +333,7 @@ jobs: path: greptimedb persist-credentials: false - name: Delete ECS and unregister runner + if: inputs.provider == 'Aliyun' uses: ./greptimedb/.github/actions/aliyun-ecs-delete with: access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} @@ -300,3 +342,20 @@ jobs: region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} instance-id: ${{ needs.provision.outputs.instance_id }} runner-name: ${{ needs.provision.outputs.runner_name }} + - uses: aws-actions/configure-aws-credentials@v4 + if: inputs.provider == 'AWS' + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + - uses: astral-sh/setup-uv@v6 + if: inputs.provider == 'AWS' + - name: Delete EC2 and unregister runner + if: inputs.provider == 'AWS' + env: + GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + AWS_REGION: ${{ vars.EC2_RUNNER_REGION }} + AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }} + AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }} + AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py diff --git a/tests/perf/test_aliyun_ecs_runner_scripts.py b/tests/perf/test_aliyun_ecs_runner_scripts.py index 53257da8732..652eae77cfe 100644 --- a/tests/perf/test_aliyun_ecs_runner_scripts.py +++ b/tests/perf/test_aliyun_ecs_runner_scripts.py @@ -30,6 +30,9 @@ from types import SimpleNamespace from unittest.mock import Mock, patch SCRIPTS_DIR = Path(__file__).parents[2] / ".github/scripts" +sys.path.insert(0, str(SCRIPTS_DIR)) + +import runner_utils def load_module(name: str, filename: str): @@ -58,7 +61,7 @@ class GitHubApiResponseTest(unittest.TestCase): response.read.return_value = body response.__enter__ = Mock(return_value=response) response.__exit__ = Mock(return_value=False) - with patch.object(provision.urllib.request, "urlopen", return_value=response): + with patch.object(runner_utils.urllib.request, "urlopen", return_value=response): self.assertEqual(provision.github_api("token", "DELETE", "/test"), expected) def test_invalid_json_still_fails(self): @@ -66,10 +69,26 @@ class GitHubApiResponseTest(unittest.TestCase): response.read.return_value = b"not json" response.__enter__ = Mock(return_value=response) response.__exit__ = Mock(return_value=False) - with patch.object(provision.urllib.request, "urlopen", return_value=response): + with patch.object(runner_utils.urllib.request, "urlopen", return_value=response): with self.assertRaises(ValueError): provision.github_api("token", "GET", "/test") + def test_http_errors_still_exit_instead_of_becoming_recoverable(self): + error = runner_utils.urllib.error.HTTPError( + "https://api.github.com/test", 403, "Forbidden", {}, io.BytesIO(b"denied") + ) + with patch.object(runner_utils.urllib.request, "urlopen", side_effect=error): + with self.assertRaisesRegex(SystemExit, "HTTP 403: denied"): + provision.github_api("token", "GET", "/test") + + def test_unregister_preserves_legacy_failure_semantics(self): + with patch.object(runner_utils, "find_runner_by_name", return_value={"id": 42}): + with patch.object(runner_utils, "github_api", side_effect=SystemExit("HTTP 403")): + with self.assertRaisesRegex(SystemExit, "HTTP 403"): + runner_utils.deregister_runner("token", "owner/repo", "runner") + with patch.object(runner_utils, "github_api", side_effect=RuntimeError("connection lost")): + self.assertFalse(runner_utils.deregister_runner("token", "owner/repo", "runner")) + class ProvisionNamingTest(unittest.TestCase): def test_runner_name_and_label_derive_from_run_id(self) -> None: diff --git a/tests/perf/test_ci_runner_scripts.py b/tests/perf/test_ci_runner_scripts.py new file mode 100644 index 00000000000..f9742079b49 --- /dev/null +++ b/tests/perf/test_ci_runner_scripts.py @@ -0,0 +1,406 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline regression coverage for CI runner selection and lifecycle.""" + +import importlib.util +import os +import re +import subprocess +import sys +import tempfile +import textwrap +import unittest +from datetime import datetime, timedelta, timezone +from pathlib import Path +from unittest.mock import Mock, patch + +SCRIPTS = Path(__file__).parents[2] / ".github/scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +config = load("ci_config_test", "ci-runner-config.py") +aws = load("aws_provision_test", "aws-ec2-runner-provision.py") +teardown = load("aws_teardown_test", "aws-ec2-runner-teardown.py") + + +def provision_config(**overrides): + return aws.ProvisionConfig(**({ + "region": "ap-southeast-1", "image_id": "ami-test", "instance_type": "c7i.2xlarge", + "subnet_id": "subnet-test", "security_group_id": "sg-test", "repo": "owner/repo", + "run_id": "123-2", "github_token": "secret", "system_disk_gib": 80, + } | overrides)) + + +def client_with_image(size=40, arch="x86_64"): + client = Mock() + client.describe_images.return_value = {"Images": [{ + "State": "available", "Architecture": arch, "RootDeviceName": "/dev/sda1", + "BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": size}}], + }]} + client.describe_instance_types.return_value = {"InstanceTypes": [{ + "ProcessorInfo": {"SupportedArchitectures": ["x86_64"]}, + }]} + client.run_instances.return_value = {"Instances": [{"InstanceId": "i-test"}]} + client.get_console_output.return_value = {"Output": "boot log"} + return client + + +class ProviderConfigTest(unittest.TestCase): + def test_defaults_and_overrides(self): + for provider, requested, expected in ( + ("Aliyun", "auto", "ecs.c9i.2xlarge"), ("AWS", "auto", "c7i.2xlarge"), + ("Aliyun", "ecs.c9i.4xlarge", "ecs.c9i.4xlarge"), + ("AWS", "c6i.4xlarge", "c6i.4xlarge"), + ("AWS", "c7i.metal-24xl", "c7i.metal-24xl"), + ): + with self.subTest(provider=provider, requested=requested): + self.assertEqual(config.resolve_instance_type(provider, requested), expected) + + def test_wrong_provider_and_shell_fragments_rejected(self): + for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"), + ("Aliyun", "c7i.2xlarge"), ("AWS", ""), + ("AWS", "c7i.2xlarge; touch /tmp/unexpected")): + with self.subTest(provider=provider, requested=requested), self.assertRaises(ValueError): + config.resolve_instance_type(provider, requested) + + def test_cli_writes_resolved_output(self): + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "output" + result = subprocess.run([sys.executable, str(SCRIPTS / "ci-runner-config.py"), + "--provider", "AWS"], + env=os.environ | {"GITHUB_OUTPUT": str(output)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), "instance_type=c7i.2xlarge\n") + + + +class BenchmarkSetupTest(unittest.TestCase): + def run_setup(self, missing_tools="", docker_installed=True, docker_ready=True): + action = (SCRIPTS.parent / "actions/setup-benchmark/action.yml").read_text() + steps = re.findall(r" run: \|\n((?: .*\n|\n)+)", action) + self.assertEqual(len(steps), 2) + script = "\n".join(textwrap.dedent(step) for step in steps) + # Execute the actual action twice with host commands stubbed: installation + # changes availability, so the second pass must perform no mutations. + prelude = r""" +function . { ID=ubuntu; VERSION_CODENAME=noble; } +uname() { echo x86_64; } +dpkg() { echo amd64; } +curl() { echo fixture-key; } +command() { + if [[ "$1" == -v ]]; then + if [[ "$2" == docker ]]; then [[ "$DOCKER_INSTALLED" == true ]]; return; fi + if [[ " $MISSING_TOOLS " == *" $2 "* ]]; then return 1; fi + return 0 + fi + builtin command "$@" +} +docker() { [[ "$DOCKER_READY" == true ]]; } +sudo() { + printf '%s\n' "$*" >> "$CALLS" + if [[ "$*" == *' apt-get install '* ]]; then + MISSING_TOOLS='' + [[ "$*" != *' docker-ce '* ]] || DOCKER_INSTALLED=true + fi + if [[ "$*" == *'systemctl enable --now docker' ]]; then DOCKER_READY=true; fi + if [[ "$*" == *' tee '* ]]; then while IFS= read -r line; do :; done; fi + return 0 +} +""" + with tempfile.TemporaryDirectory() as tmp: + calls = Path(tmp) / "calls" + calls.touch() + env = os.environ | {"CALLS": str(calls), "MISSING_TOOLS": missing_tools, + "DOCKER_INSTALLED": str(docker_installed).lower(), + "DOCKER_READY": str(docker_ready).lower()} + result = subprocess.run(["bash", "-c", prelude + script + + '\nprintf "SECOND PASS\\n" >> "$CALLS"\n' + script], + env=env, capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + first, second = calls.read_text().split("SECOND PASS\n") + self.assertEqual(second, "", "Second setup must reuse installed dependencies") + return first + + def test_prepared_runner_skips_all_privileged_changes(self): + self.assertEqual(self.run_setup(), "") + + def test_missing_tools_are_installed_without_changing_docker(self): + calls = self.run_setup(missing_tools="jq rsync") + self.assertIn("--no-install-recommends jq rsync", calls) + self.assertIn("NEEDRESTART_MODE=l", calls) + self.assertNotIn("docker", calls) + + def test_missing_docker_is_installed_once(self): + calls = self.run_setup(docker_installed=False, docker_ready=False) + self.assertIn("NEEDRESTART_MODE=l", calls) + self.assertIn("docker-ce docker-ce-cli containerd.io docker-buildx-plugin", calls) + self.assertIn("systemctl enable --now docker", calls) + + def test_existing_stopped_docker_is_started_without_installation(self): + self.assertEqual(self.run_setup(docker_ready=False), "-n systemctl enable --now docker\n") + + +class AwsProvisionTest(unittest.TestCase): + def test_host_policy_matches_aliyun_before_runner_activation(self): + aliyun = load("aliyun_host_policy_test", "aliyun-ecs-runner-provision.py") + ecs_script = aliyun.render_user_data("name", "label", "token", "owner/repo") + aws_script = aws.render_user_data("name", "token", "owner/repo", "https://example.com/runner.tgz") + start = ecs_script.index("# Mask systemd-oomd") + end = ecs_script.index("free --human", start) + len("free --human") + policy = ecs_script[start:end] + self.assertIn(policy, aws_script) + self.assertLess(aws_script.index(policy), aws_script.index("./svc.sh start")) + self.assertLess(aws_script.index("OOMPolicy=continue"), aws_script.index("./svc.sh start")) + self.assertLess(aws_script.index("./svc.sh install runner"), aws_script.index("service_name=$(cat .service)")) + self.assertIn('"/etc/systemd/system/$service_name.d/oom.conf"', aws_script) + + def test_ubuntu_image_resolution_and_override(self): + client = Mock() + client.describe_images.return_value = {"Images": [ + {"ImageId": "ami-new", "CreationDate": "2026-09-04", "Name": "noble-new"}, + {"ImageId": "ami-old", "CreationDate": "2026-08-04", "Name": "noble-old"}, + ]} + self.assertEqual(aws.resolve_image(client, "auto"), "ami-new") + request = client.describe_images.call_args.kwargs + self.assertEqual(request["Owners"], ["099720109477"]) + self.assertIn({"Name": "name", "Values": [ + "ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]}, request["Filters"]) + client.reset_mock() + self.assertEqual(aws.resolve_image(client, "ami-pinned"), "ami-pinned") + client.describe_images.assert_not_called() + client.describe_images.return_value = {"Images": []} + with self.assertRaisesRegex(ValueError, "No Canonical Ubuntu"): + aws.resolve_image(client, "auto") + + def test_disk_size_and_architecture_fail_before_allocation(self): + for client, pattern in ((client_with_image(size=500), "smaller than AMI"), + (client_with_image(arch="arm64"), "x86_64")): + with self.subTest(pattern=pattern), patch.object(aws, "create_registration_token") as token: + with self.assertRaisesRegex(ValueError, pattern): + aws.run_instance(client, provision_config()) + client.run_instances.assert_not_called() + token.assert_not_called() + + def test_arm_instance_rejected_before_registration(self): + client = client_with_image() + client.describe_instance_types.return_value["InstanceTypes"][0]["ProcessorInfo"]["SupportedArchitectures"] = ["arm64"] + with patch.object(aws, "create_registration_token") as token: + with self.assertRaisesRegex(ValueError, "x86_64"): + aws.run_instance(client, provision_config(instance_type="r8g.2xlarge")) + token.assert_not_called() + client.run_instances.assert_not_called() + + def test_create_request_and_recovery_tags(self): + client = client_with_image() + with patch.object(aws, "create_registration_token", return_value="short-lived-token"), \ + patch.object(aws, "runner_download", return_value="https://example.com/runner.tar.gz"): + self.assertEqual(aws.run_instance(client, provision_config(system_disk_gib=100)), "i-test") + request = client.run_instances.call_args.kwargs + self.assertEqual(request["BlockDeviceMappings"], [{"DeviceName": "/dev/sda1", "Ebs": { + "VolumeSize": 100, "VolumeType": "gp3", "Iops": 3000, "Throughput": 125, + "DeleteOnTermination": True}}]) + self.assertEqual(request["ClientToken"], "ci-ec2-123-2") + self.assertNotIn("InstanceMarketOptions", request) + self.assertNotIn("IamInstanceProfile", request) + for specification in request["TagSpecifications"]: + tags = {t["Key"]: t["Value"] for t in specification["Tags"]} + self.assertEqual(tags["managed-by"], "query-regression-ci") + self.assertEqual(tags[aws.REPO_TAG], "owner/repo") + self.assertEqual(tags["query-regression-run-id"], "123-2") + self.assertEqual(tags["runner-ttl-hours"], "8") + self.assertNotIn("secret", request["UserData"]) + self.assertIn("--token short-lived-token", request["UserData"]) + self.assertNotIn("apt-get install", request["UserData"]) + self.assertIn('for tool in git curl tar sudo; do command -v "$tool"; done', request["UserData"]) + self.assertLess(request["UserData"].index("usermod -aG docker runner"), + request["UserData"].index("./svc.sh start")) + self.assertIn("visudo -cf /etc/sudoers.d/ci-runner", request["UserData"]) + self.assertLess(request["UserData"].index("$nrconf{override_rc}"), + request["UserData"].index("./svc.sh start")) + check = subprocess.run(["bash", "-n"], input=request["UserData"], text=True, capture_output=True) + self.assertEqual(check.returncode, 0, check.stderr) + + def test_partial_provision_keeps_outputs_and_console(self): + client = client_with_image() + client.get_waiter.return_value.wait.side_effect = TimeoutError("not running") + with tempfile.TemporaryDirectory() as tmp: + output = Path(tmp) / "output" + with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch.object(aws, "run_instance", return_value="i-test"): + with self.assertRaises(TimeoutError): + aws.provision(client, provision_config()) + self.assertEqual(output.read_text().splitlines(), [ + "instance_id=i-test", "label=ci-ec2-123-2", + "runner_name=ci-ec2-123-2"]) + client.get_console_output.assert_called_once() + + def test_online_poll_and_timeout(self): + for online in (True, False): + with self.subTest(online=online), patch.object(aws, "run_instance", return_value="i-test"), \ + patch.object(aws, "find_runner_by_name", side_effect=[None, {"status": "online"}]), \ + patch.object(aws.time, "sleep"), \ + patch.object(aws.time, "monotonic", side_effect=[0, 1, 2 if online else 601]): + client = client_with_image() + if online: + self.assertEqual(aws.provision(client, provision_config()), 0) + else: + with self.assertRaises(TimeoutError): + aws.provision(client, provision_config()) + client.get_console_output.assert_called_once() + + +class AwsTeardownTest(unittest.TestCase): + NOW = datetime(2026, 9, 23, 10, tzinfo=timezone.utc) + + def setUp(self): + sleeper = patch.object(teardown.time, "sleep") + sleeper.start() + self.addCleanup(sleeper.stop) + + def instance(self, identity="i-test", age=9, ttl="8"): + tags = [{"Key": teardown.RUNNER_TAG, "Value": "ci-ec2-123-2"}] + if ttl is not None: + tags.append({"Key": teardown.TTL_TAG_KEY, "Value": ttl}) + return {"InstanceId": identity, "LaunchTime": self.NOW - timedelta(hours=age), "Tags": tags} + + def test_pagination_and_ownership_filters(self): + client = Mock() + client.get_paginator.return_value.paginate.return_value = [ + {"Reservations": [{"Instances": [self.instance("i-a")]}]}, + {"Reservations": [{"Instances": [self.instance("i-b")]}]}, + ] + self.assertEqual([i["InstanceId"] for i in teardown.managed_instances(client, "owner/repo", "123-2")], ["i-a", "i-b"]) + self.assertEqual(client.get_paginator.return_value.paginate.call_args.kwargs["Filters"], [ + {"Name": "tag:managed-by", "Values": ["query-regression-ci"]}, + {"Name": "tag:github-repository", "Values": ["owner/repo"]}, + {"Name": "tag:query-regression-run-id", "Values": ["123-2"]}, + ]) + + def test_sweep_expiry_and_missing_or_invalid_ttl(self): + instances = [self.instance("expired", age=8), self.instance("fresh", age=7), + self.instance("invalid", ttl="bad"), + self.instance("missing-expired", age=4, ttl=None), + self.instance("missing-fresh", age=3, ttl=None)] + with patch.object(teardown, "managed_instances", return_value=instances), \ + patch.object(teardown, "delete_instance", return_value=True) as delete, \ + patch.object(teardown, "deregister_runner", return_value=True) as unregister: + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", now=self.NOW), 0) + self.assertEqual([c.args[1] for c in delete.call_args_list], ["expired", "missing-expired"]) + self.assertEqual(unregister.call_count, 2) + + def test_ttl_decisions_match_aliyun(self): + aliyun = load("aliyun_ttl_policy_test", "aliyun-ecs-runner-teardown.py") + for age, ttl in [(3, None), (4, None), (7, "8"), (8, "8"), (9, "8"), + (10, "bad"), (10, ""), (10, "0"), (200, "169"), (168, "168")]: + instance = self.instance(age=age, ttl=ttl) + created = instance["LaunchTime"].strftime("%Y-%m-%dT%H:%M:%SZ") + expected = aliyun.expired_instance_names( + [(instance["InstanceId"], "runner", created, ttl)], self.NOW, timedelta(hours=4)) + with self.subTest(age=age, ttl=ttl), \ + patch.object(teardown, "managed_instances", return_value=[instance]), \ + patch.object(teardown, "delete_instance", return_value=True) as delete, \ + patch.object(teardown, "deregister_runner", return_value=True): + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", now=self.NOW), 0) + self.assertEqual([call.args[1] for call in delete.call_args_list], + [identity for identity, _ in expected]) + + def test_targeted_cleanup_recovers_missing_outputs_without_waiting_for_ttl(self): + with patch.object(teardown, "managed_instances", return_value=[self.instance(age=0)]), \ + patch.object(teardown, "delete_instance", return_value=True) as delete, \ + patch.object(teardown, "deregister_runner", return_value=True) as unregister: + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 0) + delete.assert_called_once() + unregister.assert_called_once_with("token", "owner/repo", "ci-ec2-123-2") + + def test_independent_cloud_and_github_cleanup_failures(self): + for cloud_ok, github_result in ((False, True), (True, RuntimeError("GitHub unavailable"))): + with self.subTest(cloud_ok=cloud_ok), \ + patch.object(teardown, "managed_instances", return_value=[self.instance()]), \ + patch.object(teardown, "delete_instance", return_value=cloud_ok) as delete, \ + patch.object(teardown, "deregister_runner", side_effect=[github_result]) as unregister: + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1) + delete.assert_called_once() + unregister.assert_called_once() + + def test_already_gone_instance_still_unregisters(self): + with patch.object(teardown, "managed_instances", return_value=[]), \ + patch.object(teardown, "deregister_runner", return_value=True) as unregister: + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 0) + unregister.assert_called_once() + + def test_retried_teardown_uses_original_instance_and_runner(self): + client = Mock() + client.get_paginator.return_value.paginate.return_value = [ + {"Reservations": [{"Instances": [self.instance()]}]}, + ] + with patch.object(teardown, "delete_instance", return_value=True) as delete, \ + patch.object(teardown, "deregister_runner", return_value=True) as unregister: + self.assertEqual(teardown.teardown(client, "owner/repo", "token", run_id="123-3", + instance_id="i-test", runner_name="ci-ec2-123-2"), 0) + filters = client.get_paginator.return_value.paginate.call_args.kwargs["Filters"] + self.assertIn({"Name": "instance-id", "Values": ["i-test"]}, filters) + self.assertFalse(any(f["Name"] == "tag:query-regression-run-id" for f in filters)) + delete.assert_called_once_with(client, "i-test") + unregister.assert_called_once_with("token", "owner/repo", "ci-ec2-123-2") + client.get_paginator.return_value.paginate.return_value = [] + with patch.object(teardown, "deregister_runner", return_value=True) as unregister: + self.assertEqual(teardown.teardown(client, "owner/repo", "token", run_id="123-3", + instance_id="i-test", runner_name="ci-ec2-123-2"), 0) + unregister.assert_called_once_with("token", "owner/repo", "ci-ec2-123-2") + + def test_terminate_waits_and_handles_not_found(self): + client = client_with_image() + self.assertTrue(teardown.delete_instance(client, "i-test")) + client.get_waiter.assert_called_with("instance_terminated") + error = RuntimeError("gone") + error.response = {"Error": {"Code": "InvalidInstanceID.NotFound"}} + client.terminate_instances.side_effect = error + with patch.object(teardown.time, "sleep"): + self.assertTrue(teardown.delete_instance(client, "i-test")) + + def test_transient_not_found_is_retried(self): + client = client_with_image() + error = RuntimeError("not visible yet") + error.response = {"Error": {"Code": "InvalidInstanceID.NotFound"}} + client.terminate_instances.side_effect = [error, {}] + self.assertTrue(teardown.delete_instance(client, "i-test")) + self.assertEqual(client.terminate_instances.call_count, 2) + + def test_missing_tags_are_polled_before_cleanup(self): + with patch.object(teardown, "managed_instances", side_effect=[[], [self.instance()]]), \ + patch.object(teardown, "delete_instance", return_value=True) as delete, \ + patch.object(teardown, "deregister_runner", return_value=True): + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 0) + delete.assert_called_once() + + def test_github_system_exit_does_not_skip_other_instances(self): + with patch.object(teardown, "managed_instances", return_value=[self.instance("i-a"), self.instance("i-b")]), \ + patch.object(teardown, "delete_instance", return_value=True) as delete, \ + patch.object(teardown, "deregister_runner", side_effect=[SystemExit("HTTP 403"), True]): + self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1) + self.assertEqual([c.args[1] for c in delete.call_args_list], ["i-a", "i-b"]) + + +if __name__ == "__main__": + unittest.main()