mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-10-12 15:02:32 +00:00
ci: add optional AWS runners for observability benchmarks
Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
WenyXu
@@ -0,0 +1,38 @@
|
||||
name: Set up benchmark dependencies
|
||||
description: Reuse installed tools and install missing dependencies on an Ubuntu self-hosted runner
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Check host and install missing tools
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ "$(uname -m)" == x86_64 ]]
|
||||
command -v git
|
||||
missing=()
|
||||
for tool in curl jq python3 rsync; do
|
||||
command -v "$tool" >/dev/null || missing+=("$tool")
|
||||
done
|
||||
if ((${#missing[@]})); then
|
||||
sudo -n apt-get update
|
||||
sudo -n env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends "${missing[@]}"
|
||||
fi
|
||||
- name: Install Docker if missing and verify access
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v docker >/dev/null; then
|
||||
. /etc/os-release
|
||||
[[ "$ID" == ubuntu ]]
|
||||
sudo -n install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo -n tee /etc/apt/keyrings/docker.asc >/dev/null
|
||||
sudo -n chmod a+r /etc/apt/keyrings/docker.asc
|
||||
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $VERSION_CODENAME stable" | sudo -n tee /etc/apt/sources.list.d/docker.list >/dev/null
|
||||
sudo -n apt-get update
|
||||
sudo -n env DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=l apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io docker-buildx-plugin
|
||||
fi
|
||||
if ! docker info >/dev/null 2>&1; then
|
||||
sudo -n systemctl enable --now docker
|
||||
fi
|
||||
# Membership must be configured before the runner starts, not in this job.
|
||||
docker info
|
||||
@@ -49,10 +49,10 @@ import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from dataclasses import dataclass
|
||||
|
||||
from runner_utils import append_github_output, create_registration_token, find_runner_by_name, github_api
|
||||
|
||||
RUNNER_NAME_PREFIX = "qreg-ecs"
|
||||
RUNNER_LABEL_PREFIX = "query-regression-ecs"
|
||||
MANAGED_BY_TAG_KEY = "managed-by"
|
||||
@@ -243,63 +243,6 @@ def encode_user_data(script: str) -> str:
|
||||
return base64.b64encode(script.encode("utf-8")).decode("ascii")
|
||||
|
||||
|
||||
def github_api(token: str, method: str, path: str, body: dict | None = None) -> dict:
|
||||
data = json.dumps(body).encode("utf-8") if body is not None else None
|
||||
request = urllib.request.Request(
|
||||
f"https://api.github.com{path}",
|
||||
data=data,
|
||||
method=method,
|
||||
headers={
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
if response.status == 204:
|
||||
return {}
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as error:
|
||||
# GitHub's error body says exactly why (e.g. "Must have admin rights to
|
||||
# Repository" for a PAT without the required scope); surface it instead
|
||||
# of a bare "HTTP Error 403".
|
||||
body = error.read().decode("utf-8", "replace")
|
||||
raise SystemExit(
|
||||
f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n"
|
||||
"The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs "
|
||||
"'repo' scope (classic PAT) or 'Administration: write' on the "
|
||||
"repository (fine-grained PAT)."
|
||||
) from error
|
||||
|
||||
|
||||
def create_registration_token(github_token: str, repo: str) -> str:
|
||||
response = github_api(
|
||||
github_token,
|
||||
"POST",
|
||||
f"/repos/{repo}/actions/runners/registration-token",
|
||||
body={},
|
||||
)
|
||||
return response["token"]
|
||||
|
||||
|
||||
def find_runner_by_name(github_token: str, repo: str, name: str) -> dict | None:
|
||||
page = 1
|
||||
while True:
|
||||
response = github_api(
|
||||
github_token,
|
||||
"GET",
|
||||
f"/repos/{repo}/actions/runners?per_page=100&page={page}",
|
||||
)
|
||||
runners = response.get("runners", [])
|
||||
for runner in runners:
|
||||
if runner.get("name") == name:
|
||||
return runner
|
||||
if len(runners) < 100:
|
||||
return None
|
||||
page += 1
|
||||
|
||||
|
||||
def make_ecs_client(config: ProvisionConfig):
|
||||
from alibabacloud_ecs20140526.client import Client as EcsClient
|
||||
from alibabacloud_tea_openapi.models import Config as OpenApiConfig
|
||||
@@ -388,13 +331,6 @@ def dump_console_output(client, region_id: str, instance_id: str) -> None:
|
||||
summary.write("\n```\n</details>\n")
|
||||
|
||||
|
||||
def append_github_output(name: str, value: str) -> None:
|
||||
output_path = os.environ.get("GITHUB_OUTPUT")
|
||||
if output_path:
|
||||
with open(output_path, "a", encoding="utf-8") as output:
|
||||
output.write(f"{name}={value}\n")
|
||||
|
||||
|
||||
def run_instance(client, config: ProvisionConfig, user_data: str) -> str:
|
||||
from alibabacloud_ecs20140526 import models as ecs_models
|
||||
|
||||
|
||||
@@ -47,6 +47,8 @@ import sys
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
from runner_utils import deregister_runner
|
||||
|
||||
# Reuse the GitHub API client, runner lookup, and tag constants from the
|
||||
# provision script.
|
||||
_PROVISION_SPEC = importlib.util.spec_from_file_location(
|
||||
@@ -159,20 +161,6 @@ def delete_instance(client, instance_id: str, region_id: str | None = None) -> b
|
||||
return False
|
||||
|
||||
|
||||
def deregister_runner(token: str, repo: str, runner_name: str) -> bool:
|
||||
runner = provision.find_runner_by_name(token, repo, runner_name)
|
||||
if runner is None:
|
||||
print(f"Runner {runner_name} is not registered", flush=True)
|
||||
return True
|
||||
try:
|
||||
provision.github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}")
|
||||
print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True)
|
||||
return True
|
||||
except Exception as error: # noqa: BLE001
|
||||
print(f"Failed to deregister runner {runner_name}: {error}", flush=True)
|
||||
return False
|
||||
|
||||
|
||||
def list_managed_instances(
|
||||
client, region_id: str
|
||||
) -> list[tuple[str, str, str, str | None]]:
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = ["boto3>=1.35,<2"]
|
||||
# ///
|
||||
|
||||
"""Provision an ephemeral EC2 CI runner from an Ubuntu x86_64 AMI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
|
||||
from runner_utils import append_github_output, create_registration_token, find_runner_by_name, github_api
|
||||
|
||||
MANAGED_BY = "query-regression-ci"
|
||||
REPO_TAG = "github-repository"
|
||||
RUN_TAG = "query-regression-run-id"
|
||||
RUNNER_TAG = "github-runner-name"
|
||||
TTL_TAG_KEY = "runner-ttl-hours"
|
||||
|
||||
|
||||
def make_client(region: str):
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
return boto3.client("ec2", region_name=region, config=Config(
|
||||
connect_timeout=10, read_timeout=30,
|
||||
retries={"mode": "standard", "max_attempts": 3},
|
||||
))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProvisionConfig:
|
||||
region: str
|
||||
image_id: str
|
||||
instance_type: str
|
||||
subnet_id: str
|
||||
security_group_id: str
|
||||
repo: str
|
||||
run_id: str
|
||||
github_token: str
|
||||
system_disk_gib: int = 500
|
||||
ttl_hours: int = 8
|
||||
|
||||
def __post_init__(self):
|
||||
if not re.fullmatch(r"[0-9]+-[0-9]+", self.run_id):
|
||||
raise ValueError("run-id must be GitHub run_id-run_attempt")
|
||||
if not 20 <= self.system_disk_gib <= 2048:
|
||||
raise ValueError("system-disk-gib must be between 20 and 2048")
|
||||
if not 1 <= self.ttl_hours <= 168:
|
||||
raise ValueError("ttl-hours must be between 1 and 168")
|
||||
|
||||
@property
|
||||
def runner_name(self) -> str:
|
||||
return f"ci-ec2-{self.run_id}"
|
||||
|
||||
|
||||
def resolve_image(client, image_id: str) -> str:
|
||||
if image_id != "auto":
|
||||
return image_id
|
||||
images = client.describe_images(
|
||||
Owners=["099720109477"],
|
||||
Filters=[
|
||||
{"Name": "name", "Values": ["ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]},
|
||||
{"Name": "state", "Values": ["available"]},
|
||||
{"Name": "architecture", "Values": ["x86_64"]},
|
||||
],
|
||||
)["Images"]
|
||||
if not images:
|
||||
raise ValueError("No Canonical Ubuntu 24.04 amd64 AMI found in this region")
|
||||
image = max(images, key=lambda image: image["CreationDate"])
|
||||
print(f"Using AMI {image['ImageId']}: {image['Name']}")
|
||||
return image["ImageId"]
|
||||
|
||||
|
||||
def root_disk(client, image_id: str, size: int) -> dict:
|
||||
images = client.describe_images(ImageIds=[image_id])["Images"]
|
||||
if len(images) != 1:
|
||||
raise ValueError(f"AMI not found: {image_id}")
|
||||
image = images[0]
|
||||
if image["State"] != "available" or image["Architecture"] != "x86_64":
|
||||
raise ValueError("Runner AMI must be available and x86_64")
|
||||
root = next((d for d in image["BlockDeviceMappings"]
|
||||
if d["DeviceName"] == image["RootDeviceName"] and "Ebs" in d), None)
|
||||
if root is None:
|
||||
raise ValueError("Runner AMI must have an EBS root disk")
|
||||
minimum = root["Ebs"]["VolumeSize"]
|
||||
if size < minimum:
|
||||
raise ValueError(f"Requested root disk {size} GiB is smaller than AMI minimum {minimum} GiB")
|
||||
return {"DeviceName": image["RootDeviceName"], "Ebs": {
|
||||
"VolumeSize": size, "VolumeType": "gp3", "Iops": 3000,
|
||||
"Throughput": 125, "DeleteOnTermination": True,
|
||||
}}
|
||||
|
||||
|
||||
def runner_download(token: str, repo: str) -> str:
|
||||
runners = github_api(token, "GET", f"/repos/{repo}/actions/runners/downloads")
|
||||
return next(r["download_url"] for r in runners if r["os"] == "linux" and r["architecture"] == "x64")
|
||||
|
||||
|
||||
def render_user_data(name: str, token: str, repo: str, download_url: str) -> str:
|
||||
# Quote every external value before embedding it in the root shell script.
|
||||
name, token, repo, download_url = map(shlex.quote, (name, token, repo, download_url))
|
||||
return f"""#!/bin/bash
|
||||
set -euo pipefail
|
||||
for tool in git curl tar sudo; do command -v "$tool"; done
|
||||
# Mask systemd-oomd before enabling swap: Ubuntu 24.04 kills the whole
|
||||
# service cgroup on PSI pressure, and swap thrashing looks like pressure.
|
||||
systemctl disable --now systemd-oomd.socket systemd-oomd.service || true
|
||||
systemctl mask systemd-oomd.socket systemd-oomd.service || true
|
||||
# apt-daily-upgrade / unattended-upgrades can `systemctl restart` services
|
||||
# whose libraries were updated. That SIGTERM-s the runner mid-job; GitHub
|
||||
# records UserCancelled (job still valid, child exit 143). These VMs live
|
||||
# ~1h; security updates belong in the image, not at job time.
|
||||
systemctl disable --now unattended-upgrades.service apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service || true
|
||||
systemctl mask unattended-upgrades.service apt-daily.timer apt-daily-upgrade.timer apt-daily.service apt-daily-upgrade.service || true
|
||||
systemctl stop unattended-upgrades.service || true
|
||||
killall -9 unattended-upgr || true
|
||||
cat > /etc/apt/apt.conf.d/99disable-auto-updates <<'APTEOF'
|
||||
APT::Periodic::Update-Package-Lists "0";
|
||||
APT::Periodic::Unattended-Upgrade "0";
|
||||
APT::Periodic::Download-Upgradeable-Packages "0";
|
||||
APTEOF
|
||||
if [[ ! -f "/swapfile" ]]; then
|
||||
fallocate --length 16G "/swapfile"
|
||||
chmod 600 "/swapfile"
|
||||
mkswap "/swapfile"
|
||||
fi
|
||||
swapon "/swapfile"
|
||||
sysctl --write vm.swappiness=10
|
||||
swapon --show
|
||||
free --human
|
||||
|
||||
id runner >/dev/null 2>&1 || useradd --create-home --shell /bin/bash runner
|
||||
# Establish membership before the runner service starts; setup installs Docker later.
|
||||
getent group docker >/dev/null || groupadd --system docker
|
||||
usermod -aG docker runner
|
||||
printf 'runner ALL=(ALL) NOPASSWD: ALL\\n' > /etc/sudoers.d/ci-runner
|
||||
chmod 0440 /etc/sudoers.d/ci-runner
|
||||
visudo -cf /etc/sudoers.d/ci-runner
|
||||
install -d -o runner -g "$(id -gn runner)" /home/runner/benchmark-data /opt/ci-github-runner
|
||||
cd /opt/ci-github-runner
|
||||
curl --fail --location --retry 3 {download_url} | tar xz
|
||||
./bin/installdependencies.sh
|
||||
chown -R runner:"$(id -gn runner)" /opt/ci-github-runner
|
||||
runuser -u runner -- ./config.sh --unattended --ephemeral --disableupdate --url https://github.com/{repo} --token {token} --name {name} --labels {name}
|
||||
./svc.sh install runner
|
||||
# Match the ECS runner's behavior when a child process is OOM-killed.
|
||||
service_name=$(cat .service)
|
||||
mkdir -p "/etc/systemd/system/$service_name.d"
|
||||
cat > "/etc/systemd/system/$service_name.d/oom.conf" <<'EOF'
|
||||
[Service]
|
||||
OOMPolicy=continue
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
# Package installation in a job must not restart its runner service.
|
||||
install -d /etc/needrestart/conf.d
|
||||
cat > /etc/needrestart/conf.d/actions_runner_services.conf <<'EOF'
|
||||
$nrconf{{override_rc}}{{qr(^actions\\.runner\\..+\\.service$)}} = 0;
|
||||
EOF
|
||||
./svc.sh start
|
||||
"""
|
||||
|
||||
|
||||
def run_instance(client, config: ProvisionConfig) -> str:
|
||||
image_id = resolve_image(client, config.image_id)
|
||||
disk = root_disk(client, image_id, config.system_disk_gib)
|
||||
instance = client.describe_instance_types(InstanceTypes=[config.instance_type])["InstanceTypes"][0]
|
||||
if "x86_64" not in instance["ProcessorInfo"]["SupportedArchitectures"]:
|
||||
raise ValueError("Runner instance type must support x86_64")
|
||||
token = create_registration_token(config.github_token, config.repo)
|
||||
print(f"::add-mask::{token}")
|
||||
script = render_user_data(config.runner_name, token, config.repo, runner_download(config.github_token, config.repo))
|
||||
tags = [
|
||||
{"Key": "Name", "Value": config.runner_name},
|
||||
{"Key": "managed-by", "Value": MANAGED_BY},
|
||||
{"Key": REPO_TAG, "Value": config.repo},
|
||||
{"Key": RUN_TAG, "Value": config.run_id},
|
||||
{"Key": RUNNER_TAG, "Value": config.runner_name},
|
||||
{"Key": TTL_TAG_KEY, "Value": str(config.ttl_hours)},
|
||||
]
|
||||
response = client.run_instances(
|
||||
ImageId=image_id, InstanceType=config.instance_type,
|
||||
MinCount=1, MaxCount=1, SubnetId=config.subnet_id,
|
||||
SecurityGroupIds=[config.security_group_id],
|
||||
BlockDeviceMappings=[disk], UserData=script,
|
||||
ClientToken=config.runner_name,
|
||||
MetadataOptions={"HttpTokens": "required", "HttpEndpoint": "enabled"},
|
||||
TagSpecifications=[{"ResourceType": kind, "Tags": tags} for kind in ("instance", "volume")],
|
||||
)
|
||||
return response["Instances"][0]["InstanceId"]
|
||||
|
||||
|
||||
def dump_console(client, instance_id: str) -> None:
|
||||
try:
|
||||
# Boto3 decodes the API's base64 console output.
|
||||
output = client.get_console_output(InstanceId=instance_id, Latest=True).get("Output", "")
|
||||
print(f"::group::EC2 console {instance_id}")
|
||||
print("\n".join(output.splitlines()[-80:]))
|
||||
print("::endgroup::")
|
||||
except Exception as error:
|
||||
print(f"Unable to read console for {instance_id}: {error}")
|
||||
|
||||
|
||||
def provision(client, config: ProvisionConfig) -> int:
|
||||
instance_id = run_instance(client, config)
|
||||
# Publish before waiting: failed registration must still reach teardown.
|
||||
for key, value in (("instance_id", instance_id),
|
||||
("label", config.runner_name), ("runner_name", config.runner_name)):
|
||||
append_github_output(key, value)
|
||||
try:
|
||||
client.get_waiter("instance_running").wait(
|
||||
InstanceIds=[instance_id], WaiterConfig={"Delay": 5, "MaxAttempts": 60})
|
||||
deadline = time.monotonic() + 10 * 60
|
||||
while time.monotonic() < deadline:
|
||||
runner = find_runner_by_name(config.github_token, config.repo, config.runner_name)
|
||||
if runner and runner.get("status") == "online":
|
||||
print(f"Runner {config.runner_name} online: {instance_id}")
|
||||
return 0
|
||||
time.sleep(5)
|
||||
raise TimeoutError(f"Runner {config.runner_name} did not come online")
|
||||
finally:
|
||||
dump_console(client, instance_id)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name, env in (("region", "AWS_REGION"), ("repo", "GITHUB_REPOSITORY"),
|
||||
("github-token", "GH_PERSONAL_ACCESS_TOKEN"),
|
||||
("run-id", "AWS_EC2_RUN_ID"), ("image-id", "AWS_EC2_IMAGE_ID"),
|
||||
("instance-type", "AWS_EC2_INSTANCE_TYPE"),
|
||||
("subnet-id", "AWS_EC2_SUBNET_ID"),
|
||||
("security-group-id", "AWS_EC2_SECURITY_GROUP_ID")):
|
||||
parser.add_argument(f"--{name}", default=os.environ.get(env) or ("auto" if name == "image-id" else None))
|
||||
parser.add_argument("--system-disk-gib", type=int, default=os.environ.get("AWS_EC2_SYSTEM_DISK_GIB", "500"))
|
||||
parser.add_argument("--ttl-hours", type=int, default=os.environ.get("AWS_EC2_TTL_HOURS", "8"))
|
||||
args = parser.parse_args()
|
||||
for name, value in vars(args).items():
|
||||
if value is None or value == "":
|
||||
parser.error(f"Missing {name.replace('_', '-')}")
|
||||
config = ProvisionConfig(**vars(args))
|
||||
return provision(make_client(args.region), config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# /// script
|
||||
# requires-python = ">=3.10"
|
||||
# dependencies = ["boto3>=1.35,<2"]
|
||||
# ///
|
||||
|
||||
"""Delete managed EC2 CI runners, or sweep expired instances by ownership tags."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from runner_utils import deregister_runner
|
||||
|
||||
_SPEC = importlib.util.spec_from_file_location("aws_ec2_runner_provision", Path(__file__).with_name("aws-ec2-runner-provision.py"))
|
||||
assert _SPEC is not None and _SPEC.loader is not None
|
||||
provision = importlib.util.module_from_spec(_SPEC)
|
||||
sys.modules[_SPEC.name] = provision
|
||||
_SPEC.loader.exec_module(provision)
|
||||
make_client = provision.make_client
|
||||
dump_console = provision.dump_console
|
||||
MANAGED_BY = provision.MANAGED_BY
|
||||
REPO_TAG = provision.REPO_TAG
|
||||
RUN_TAG = provision.RUN_TAG
|
||||
RUNNER_TAG = provision.RUNNER_TAG
|
||||
TTL_TAG_KEY = provision.TTL_TAG_KEY
|
||||
SWEEP_TTL_HOURS = 4
|
||||
|
||||
|
||||
def managed_instances(client, repo: str, run_id: str | None = None,
|
||||
instance_id: str | None = None) -> list[dict]:
|
||||
filters = [{"Name": "tag:managed-by", "Values": [MANAGED_BY]},
|
||||
{"Name": f"tag:{REPO_TAG}", "Values": [repo]}]
|
||||
if instance_id:
|
||||
# A retried teardown may have a newer workflow attempt than creation.
|
||||
filters.append({"Name": "instance-id", "Values": [instance_id]})
|
||||
elif run_id:
|
||||
filters.append({"Name": f"tag:{RUN_TAG}", "Values": [run_id]})
|
||||
return [instance
|
||||
for page in client.get_paginator("describe_instances").paginate(Filters=filters)
|
||||
for reservation in page["Reservations"] for instance in reservation["Instances"]]
|
||||
|
||||
|
||||
def delete_instance(client, instance_id: str) -> bool:
|
||||
dump_console(client, instance_id)
|
||||
for attempt in range(12):
|
||||
try:
|
||||
client.terminate_instances(InstanceIds=[instance_id])
|
||||
client.get_waiter("instance_terminated").wait(
|
||||
InstanceIds=[instance_id], WaiterConfig={"Delay": 5, "MaxAttempts": 60})
|
||||
return True
|
||||
except Exception as error:
|
||||
code = getattr(error, "response", {}).get("Error", {}).get("Code")
|
||||
if code == "InvalidInstanceID.NotFound":
|
||||
if attempt < 11:
|
||||
time.sleep(5)
|
||||
continue
|
||||
# The ownership lookup observed this instance. Treat sustained
|
||||
# NotFound after bounded retries as an already-deleted VM.
|
||||
return True
|
||||
print(f"Failed to terminate {instance_id}: {error}")
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def teardown(client, repo: str, token: str, run_id: str | None = None,
|
||||
instance_id: str | None = None, now: datetime | None = None,
|
||||
runner_name: str | None = None) -> int:
|
||||
# Lookup by ownership/run tags also recovers a create whose response or job
|
||||
# outputs were lost. Sweep never touches legacy build/JSONBench instances.
|
||||
instances = managed_instances(client, repo, run_id, instance_id)
|
||||
# Tags can lag RunInstances. Retry before concluding a targeted VM is gone.
|
||||
if run_id and not instances:
|
||||
for _ in range(11):
|
||||
time.sleep(5)
|
||||
instances = managed_instances(client, repo, run_id, instance_id)
|
||||
if instances:
|
||||
break
|
||||
ok = True
|
||||
found = False
|
||||
for instance in instances:
|
||||
identity = instance["InstanceId"]
|
||||
if instance_id and identity != instance_id:
|
||||
continue
|
||||
found = True
|
||||
tags = {t["Key"]: t["Value"] for t in instance.get("Tags", [])}
|
||||
name = tags.get(RUNNER_TAG, "")
|
||||
if not name.startswith("ci-ec2-"):
|
||||
print(f"Skipping {identity}: missing managed runner name")
|
||||
ok = False
|
||||
continue
|
||||
if not run_id:
|
||||
ttl = tags.get(TTL_TAG_KEY)
|
||||
try:
|
||||
hours = SWEEP_TTL_HOURS if ttl is None else int(ttl)
|
||||
if not 1 <= hours <= 168:
|
||||
raise ValueError("TTL outside 1..168 hours")
|
||||
except (TypeError, ValueError):
|
||||
print(f"Skipping {identity}: invalid TTL {ttl!r}")
|
||||
continue
|
||||
if (now or datetime.now(timezone.utc)) - instance["LaunchTime"] < timedelta(hours=hours):
|
||||
continue
|
||||
# A GitHub outage must not prevent EC2 deletion, or vice versa.
|
||||
ok &= delete_instance(client, identity)
|
||||
try:
|
||||
ok &= deregister_runner(token, repo, name)
|
||||
except (Exception, SystemExit) as error:
|
||||
print(f"Failed to unregister {name}: {error}")
|
||||
ok = False
|
||||
if run_id and not found:
|
||||
# EC2 may already be gone while its GitHub registration remains.
|
||||
name = runner_name or f"ci-ec2-{run_id}"
|
||||
if not name.startswith("ci-ec2-"):
|
||||
raise ValueError("Invalid CI runner name")
|
||||
ok &= deregister_runner(token, repo, name)
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
for name, env in (("region", "AWS_REGION"), ("repo", "GITHUB_REPOSITORY"),
|
||||
("github-token", "GH_PERSONAL_ACCESS_TOKEN"),
|
||||
("run-id", "AWS_EC2_RUN_ID"), ("instance-id", "AWS_EC2_INSTANCE_ID"),
|
||||
("runner-name", "AWS_EC2_RUNNER_NAME")):
|
||||
parser.add_argument(f"--{name}", default=os.environ.get(env))
|
||||
parser.add_argument("--sweep", action="store_true")
|
||||
args = parser.parse_args()
|
||||
for name in ("region", "repo", "github_token"):
|
||||
if not getattr(args, name):
|
||||
parser.error(f"Missing {name.replace('_', '-')}")
|
||||
if not args.sweep and not args.run_id:
|
||||
parser.error("Missing run-id for targeted teardown")
|
||||
return teardown(make_client(args.region), args.repo, args.github_token,
|
||||
None if args.sweep else args.run_id,
|
||||
None if args.sweep else args.instance_id,
|
||||
runner_name=None if args.sweep else args.runner_name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Resolve dispatch provider and instance type before allocating a runner."""
|
||||
|
||||
import argparse
|
||||
import re
|
||||
|
||||
from runner_utils import append_github_output
|
||||
|
||||
DEFAULT_INSTANCE_TYPES = {"Aliyun": "ecs.c9i.2xlarge", "AWS": "c7i.2xlarge"}
|
||||
|
||||
|
||||
def resolve_instance_type(provider: str, requested: str) -> str:
|
||||
if provider not in DEFAULT_INSTANCE_TYPES:
|
||||
raise ValueError("provider must be Aliyun or AWS")
|
||||
instance_type = DEFAULT_INSTANCE_TYPES[provider] if requested == "auto" else requested
|
||||
pattern = r"ecs\.[a-z0-9-]+\.[a-z0-9-]+" if provider == "Aliyun" else r"[a-z0-9-]+\.[a-z0-9-]+"
|
||||
if not re.fullmatch(pattern, instance_type):
|
||||
raise ValueError(f"Invalid {provider} instance type: {instance_type!r}")
|
||||
return instance_type
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--provider", required=True)
|
||||
parser.add_argument("--instance-type", default="auto")
|
||||
args = parser.parse_args()
|
||||
instance_type = resolve_instance_type(args.provider, args.instance_type)
|
||||
append_github_output("instance_type", instance_type)
|
||||
print(f"CI runner: {args.provider} / {instance_type}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
# CI runners
|
||||
|
||||
The agent observability and long-range benchmark workflows select `Aliyun` or
|
||||
`AWS` through the `provider` dispatch input. `instance_type: auto` selects
|
||||
`ecs.c9i.2xlarge` or `c7i.2xlarge`, respectively (8 vCPU, 16 GiB).
|
||||
Existing build/release and JSONBench workflows keep their current runner actions.
|
||||
|
||||
Aliyun uses the existing ECS actions and image. Its provision and teardown
|
||||
scripts share only GitHub API and output helpers from `runner_utils.py` with
|
||||
the new AWS scripts.
|
||||
|
||||
AWS uses `aws-ec2-runner-provision.py` and `aws-ec2-runner-teardown.py` directly.
|
||||
Repository variables supply `EC2_RUNNER_REGION`, `EC2_RUNNER_SUBNET_ID`, and
|
||||
`EC2_RUNNER_SECURITY_GROUP_ID`; credentials use `AWS_ACCESS_KEY_ID`,
|
||||
`AWS_SECRET_ACCESS_KEY`, and `GH_PERSONAL_ACCESS_TOKEN` secrets.
|
||||
The current region is Singapore (`ap-southeast-1`). By default the script selects
|
||||
the newest available Canonical Ubuntu 24.04 amd64 server AMI in that region
|
||||
(owner `099720109477`) and logs its ID. Set `BENCHMARK_EC2_IMAGE_ID` to pin a
|
||||
prepared Ubuntu AMI. Existing build CI's `EC2_RUNNER_LINUX_AMD64_IMAGE_ID` is
|
||||
unaffected. Both benchmark workflows default to 500 GiB for both providers;
|
||||
explicit disk overrides must be at least the selected AMI's root snapshot size.
|
||||
The official 24.04 image does not have the old custom AMI's 500 GiB minimum.
|
||||
|
||||
AWS user data registers a fresh ephemeral runner. Git, curl, tar, and sudo are
|
||||
expected in the Ubuntu server image. Docker group membership is set before the
|
||||
runner service starts, and this dedicated VM grants the runner passwordless sudo
|
||||
so its job can install dependencies. Like ECS, the host creates 16 GiB swap with
|
||||
swappiness 10, disables systemd-oomd and background automatic updates, and sets
|
||||
`OOMPolicy=continue` on the runner service. Explicit setup package installation
|
||||
remains enabled.
|
||||
|
||||
Both benchmark jobs call `.github/actions/setup-benchmark` after checkout. It
|
||||
reuses existing tools, installs missing curl/jq/python3/rsync, installs Docker CE
|
||||
if absent, and checks Docker access. Missing packages require passwordless sudo;
|
||||
Aliyun's prepared image normally provides the tools and Docker already. Existing
|
||||
Docker installations are preserved, so exact versions need not match between
|
||||
providers. Provisioning must establish Docker permissions before starting the
|
||||
runner; adding a group inside the running job would not update its groups.
|
||||
|
||||
The subnet must provide outbound access to Ubuntu and Docker packages, GitHub,
|
||||
and the configured container registries. The IAM identity needs EC2
|
||||
image/instance-type discovery, instance and volume tagging, launch, describe,
|
||||
console output, and termination permissions. GitHub credentials need repository
|
||||
runner administration access.
|
||||
|
||||
Creation publishes instance identity before waiting for registration. Teardown
|
||||
can recover missing outputs using repository and workflow-attempt tags. The
|
||||
daily AWS janitor (03:23 UTC) removes expired instances tagged `managed-by=query-regression-ci`
|
||||
and unregisters their runners. Both providers use the existing
|
||||
`managed-by=query-regression-ci`, `query-regression-run-id`, and
|
||||
`runner-ttl-hours` tags. AWS retains a run-attempt suffix in its run identity to
|
||||
isolate retries, plus repository and runner-name tags to restrict cleanup.
|
||||
Legacy build instances do not match these ownership filters.
|
||||
Both janitors use a 4-hour fallback when the TTL tag is absent and skip invalid
|
||||
TTL tags. Benchmark jobs explicitly tag their configured TTL (default 8 hours).
|
||||
Cleanup does not check whether the GitHub job is still running. Expired instances
|
||||
are removed on the next daily sweep; normal workflow teardown runs immediately.
|
||||
TTL is measured from instance launch, so it must cover provisioning and the
|
||||
benchmark. The scheduled janitor becomes active after landing on the default
|
||||
branch.
|
||||
|
||||
Local validation:
|
||||
|
||||
```sh
|
||||
python3 tests/perf/test_ci_runner_scripts.py
|
||||
python3 tests/perf/test_aliyun_ecs_runner_scripts.py
|
||||
actionlint .github/workflows/agent-observability.yml .github/workflows/vmbench-long-range.yml .github/workflows/aws-ec2-runner-janitor.yml
|
||||
```
|
||||
|
||||
A real AWS dispatch is still needed to verify AMI boot, package
|
||||
installation, registry connectivity, and end-to-end benchmark execution.
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Small shared GitHub runner API and workflow-output helpers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
|
||||
def github_api(token: str, method: str, path: str, body: dict | None = None) -> dict:
|
||||
data = json.dumps(body).encode("utf-8") if body is not None else None
|
||||
request = urllib.request.Request(
|
||||
f"https://api.github.com{path}",
|
||||
data=data,
|
||||
method=method,
|
||||
headers={
|
||||
"Accept": "application/vnd.github+json",
|
||||
"Authorization": f"Bearer {token}",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=30) as response:
|
||||
if response.status == 204:
|
||||
return {}
|
||||
return json.loads(response.read().decode("utf-8"))
|
||||
except urllib.error.HTTPError as error:
|
||||
# GitHub's error body says exactly why (e.g. "Must have admin rights to
|
||||
# Repository" for a PAT without the required scope); surface it instead
|
||||
# of a bare "HTTP Error 403".
|
||||
body = error.read().decode("utf-8", "replace")
|
||||
raise SystemExit(
|
||||
f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n"
|
||||
"The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs "
|
||||
"'repo' scope (classic PAT) or 'Administration: write' on the "
|
||||
"repository (fine-grained PAT)."
|
||||
) from error
|
||||
|
||||
|
||||
def create_registration_token(github_token: str, repo: str) -> str:
|
||||
response = github_api(
|
||||
github_token,
|
||||
"POST",
|
||||
f"/repos/{repo}/actions/runners/registration-token",
|
||||
body={},
|
||||
)
|
||||
return response["token"]
|
||||
|
||||
|
||||
def find_runner_by_name(github_token: str, repo: str, name: str) -> dict | None:
|
||||
page = 1
|
||||
while True:
|
||||
response = github_api(
|
||||
github_token,
|
||||
"GET",
|
||||
f"/repos/{repo}/actions/runners?per_page=100&page={page}",
|
||||
)
|
||||
runners = response.get("runners", [])
|
||||
for runner in runners:
|
||||
if runner.get("name") == name:
|
||||
return runner
|
||||
if len(runners) < 100:
|
||||
return None
|
||||
page += 1
|
||||
|
||||
|
||||
def append_github_output(name: str, value: str) -> None:
|
||||
output_path = os.environ.get("GITHUB_OUTPUT")
|
||||
if output_path:
|
||||
with open(output_path, "a", encoding="utf-8") as output:
|
||||
output.write(f"{name}={value}\n")
|
||||
|
||||
|
||||
def deregister_runner(token: str, repo: str, runner_name: str) -> bool:
|
||||
runner = find_runner_by_name(token, repo, runner_name)
|
||||
if runner is None:
|
||||
print(f"Runner {runner_name} is not registered", flush=True)
|
||||
return True
|
||||
try:
|
||||
github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}")
|
||||
print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True)
|
||||
return True
|
||||
except Exception as error: # noqa: BLE001
|
||||
print(f"Failed to deregister runner {runner_name}: {error}", flush=True)
|
||||
return False
|
||||
@@ -50,15 +50,21 @@ on:
|
||||
type: string
|
||||
default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0
|
||||
required: true
|
||||
ecs_instance_type:
|
||||
description: ECS instance type (independent of Query Regression)
|
||||
provider:
|
||||
description: Cloud provider for the benchmark runner
|
||||
type: choice
|
||||
options: [Aliyun, AWS]
|
||||
default: Aliyun
|
||||
required: true
|
||||
instance_type:
|
||||
description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
|
||||
type: string
|
||||
default: ecs.c9i.2xlarge
|
||||
default: auto
|
||||
required: true
|
||||
system_disk_gib:
|
||||
description: System disk GiB; 0 = auto (S 80, P 100, M 500), or 20..2048
|
||||
description: System disk GiB (default 500; 0 uses default)
|
||||
type: number
|
||||
default: 0
|
||||
default: 500
|
||||
required: true
|
||||
benchmark_timeout_minutes:
|
||||
description: Benchmark job timeout in minutes (1..360)
|
||||
@@ -89,7 +95,8 @@ env:
|
||||
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
|
||||
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image }}
|
||||
ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
|
||||
JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
|
||||
@@ -100,6 +107,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
instance_type: ${{ steps.runner.outputs.instance_type }}
|
||||
targets: ${{ steps.inputs.outputs.targets }}
|
||||
system_disk_gib: ${{ steps.inputs.outputs.system_disk_gib }}
|
||||
greptimedb: ${{ steps.inputs.outputs.greptimedb }}
|
||||
@@ -110,9 +118,13 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Test ECS lifecycle helpers
|
||||
- name: Resolve CI runner
|
||||
id: runner
|
||||
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
|
||||
- name: Test CI runner helpers
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
|
||||
python3 greptimedb/tests/perf/test_ci_runner_scripts.py
|
||||
python3 greptimedb/tests/perf/test_agent_observability_summary.py
|
||||
- name: Validate requested benchmark
|
||||
id: inputs
|
||||
@@ -124,14 +136,11 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Validate cloud lifecycle budgets before provisioning; runtime owns workload validation.
|
||||
[[ -n "$ECS_INSTANCE_TYPE" ]]
|
||||
case "$PROFILE" in
|
||||
S) auto_disk_gib=80 ;;
|
||||
P) auto_disk_gib=100 ;;
|
||||
M) auto_disk_gib=500 ;;
|
||||
S|P|M) ;;
|
||||
*) echo "Unknown dataset: $PROFILE" >&2; exit 1 ;;
|
||||
esac
|
||||
[[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=$auto_disk_gib
|
||||
[[ "$SYSTEM_DISK_GIB" != 0 ]] || SYSTEM_DISK_GIB=500
|
||||
for value in "$SYSTEM_DISK_GIB" "$BENCHMARK_TIMEOUT_MINUTES" "$JANITOR_TTL_HOURS"; do
|
||||
[[ "$value" =~ ^[1-9][0-9]{0,3}$ ]] || { echo 'Resource budgets must be positive integers' >&2; exit 1; }
|
||||
done
|
||||
@@ -160,15 +169,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
outputs:
|
||||
label: ${{ steps.ecs.outputs.label }}
|
||||
instance_id: ${{ steps.ecs.outputs.instance_id }}
|
||||
runner_name: ${{ steps.ecs.outputs.runner_name }}
|
||||
label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
|
||||
instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
|
||||
runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Create ephemeral ECS runner
|
||||
if: inputs.provider == 'Aliyun'
|
||||
id: ecs
|
||||
uses: ./greptimedb/.github/actions/aliyun-ecs-create
|
||||
with:
|
||||
@@ -179,13 +189,35 @@ jobs:
|
||||
vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
|
||||
security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
|
||||
image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
|
||||
instance-type: ${{ env.ECS_INSTANCE_TYPE }}
|
||||
instance-type: ${{ needs.validate.outputs.instance_type }}
|
||||
enable-docker: 'true'
|
||||
system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
|
||||
ttl-hours: ${{ inputs.janitor_ttl_hours }}
|
||||
resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
|
||||
runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
|
||||
runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
if: inputs.provider == 'AWS'
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ vars.EC2_RUNNER_REGION }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
if: inputs.provider == 'AWS'
|
||||
- name: Create ephemeral EC2 runner
|
||||
if: inputs.provider == 'AWS'
|
||||
id: aws
|
||||
env:
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
|
||||
AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
|
||||
AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
|
||||
AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
|
||||
AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
|
||||
AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
|
||||
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
|
||||
benchmark:
|
||||
needs:
|
||||
- validate
|
||||
@@ -193,6 +225,7 @@ jobs:
|
||||
runs-on: ${{ needs.provision.outputs.label }}
|
||||
timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
|
||||
env:
|
||||
INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
|
||||
TARGETS: ${{ needs.validate.outputs.targets }}
|
||||
SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
|
||||
OWNER: o11ybench-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
@@ -209,6 +242,8 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Set up benchmark dependencies
|
||||
uses: ./greptimedb/.github/actions/setup-benchmark
|
||||
- name: Check Docker on ephemeral Linux host
|
||||
shell: bash
|
||||
run: |
|
||||
@@ -252,9 +287,9 @@ jobs:
|
||||
done
|
||||
jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
|
||||
--arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
|
||||
--arg instance_type "$ECS_INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
|
||||
--arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
|
||||
--argjson timeout_minutes "$BENCHMARK_TIMEOUT_MINUTES" --argjson ttl_hours "$JANITOR_TTL_HOURS" \
|
||||
'{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ecs:{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
|
||||
'{o11ybench_sha:$sha, targets:$targets, profile:$profile, db_cpus:$cpus, db_memory:$memory, images:$images, ($provider | if . == "AWS" then "ec2" else "ecs" end):{instance_type:$instance_type, system_disk_gib:$disk_gib, benchmark_timeout_minutes:$timeout_minutes, janitor_ttl_hours:$ttl_hours}}' > "$MANIFEST_PATH"
|
||||
- name: Checkout benchmark tools
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -343,7 +378,7 @@ jobs:
|
||||
needs:
|
||||
- provision
|
||||
- benchmark
|
||||
if: ${{ always() && needs.provision.outputs.instance_id != '' }}
|
||||
if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
@@ -352,6 +387,7 @@ jobs:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Delete ECS and unregister runner
|
||||
if: inputs.provider == 'Aliyun'
|
||||
uses: ./greptimedb/.github/actions/aliyun-ecs-delete
|
||||
with:
|
||||
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
@@ -360,3 +396,20 @@ jobs:
|
||||
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
instance-id: ${{ needs.provision.outputs.instance_id }}
|
||||
runner-name: ${{ needs.provision.outputs.runner_name }}
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
if: inputs.provider == 'AWS'
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ vars.EC2_RUNNER_REGION }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
if: inputs.provider == 'AWS'
|
||||
- name: Delete EC2 and unregister runner
|
||||
if: inputs.provider == 'AWS'
|
||||
env:
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
|
||||
AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
|
||||
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
name: AWS EC2 Runner Janitor
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: '23 3 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
sweep:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ vars.EC2_RUNNER_REGION }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
- name: Sweep expired CI instances and unregister runners
|
||||
env:
|
||||
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
run: uv run .github/scripts/aws-ec2-runner-teardown.py --sweep
|
||||
@@ -97,6 +97,7 @@ jobs:
|
||||
python3 tests/perf/test_query_regression_nightly_refs.py
|
||||
python3 tests/perf/test_query_regression_slash.py
|
||||
python3 tests/perf/test_aliyun_ecs_runner_scripts.py
|
||||
python3 tests/perf/test_ci_runner_scripts.py
|
||||
python3 tests/perf/test_agent_observability_summary.py
|
||||
|
||||
check:
|
||||
|
||||
@@ -35,10 +35,16 @@ on:
|
||||
type: string
|
||||
default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29
|
||||
required: true
|
||||
ecs_instance_type:
|
||||
description: ECS instance type (independent of Query Regression)
|
||||
provider:
|
||||
description: Cloud provider for the benchmark runner
|
||||
type: choice
|
||||
options: [Aliyun, AWS]
|
||||
default: Aliyun
|
||||
required: true
|
||||
instance_type:
|
||||
description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
|
||||
type: string
|
||||
default: ecs.c9i.2xlarge
|
||||
default: auto
|
||||
required: true
|
||||
benchmark_root:
|
||||
description: Absolute disk-backed work directory (outside the o11ybench checkout)
|
||||
@@ -46,9 +52,9 @@ on:
|
||||
default: /home/runner/benchmark-data
|
||||
required: true
|
||||
system_disk_gib:
|
||||
description: System disk GiB (20..2048)
|
||||
description: System disk GiB (20..2048, default 500)
|
||||
type: number
|
||||
default: 80
|
||||
default: 500
|
||||
required: true
|
||||
benchmark_timeout_minutes:
|
||||
description: Benchmark job timeout in minutes (1..360)
|
||||
@@ -78,7 +84,8 @@ env:
|
||||
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
|
||||
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image }}
|
||||
ECS_INSTANCE_TYPE: ${{ inputs.ecs_instance_type }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
BENCHMARK_TIMEOUT_MINUTES: ${{ inputs.benchmark_timeout_minutes }}
|
||||
JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
|
||||
@@ -89,6 +96,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
instance_type: ${{ steps.runner.outputs.instance_type }}
|
||||
system_disk_gib: ${{ steps.plan.outputs.system_disk_gib }}
|
||||
runtime_image: ${{ steps.plan.outputs.runtime_image }}
|
||||
o11ybench_ref: ${{ steps.plan.outputs.o11ybench_ref }}
|
||||
@@ -99,8 +107,13 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Test existing ECS lifecycle helpers
|
||||
run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
|
||||
- name: Resolve CI runner
|
||||
id: runner
|
||||
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
|
||||
- name: Test CI runner helpers
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
|
||||
python3 greptimedb/tests/perf/test_ci_runner_scripts.py
|
||||
- name: Validate budgets and build warm plan
|
||||
id: plan
|
||||
env:
|
||||
@@ -108,7 +121,6 @@ jobs:
|
||||
RUN_VICTORIAMETRICS: ${{ inputs.run_victoriametrics }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
[[ -n "$ECS_INSTANCE_TYPE" ]]
|
||||
[[ "$BENCHMARK_ROOT" == /* && "$BENCHMARK_ROOT" != / && "$BENCHMARK_ROOT" != */ ]]
|
||||
[[ "$BENCHMARK_ROOT" != /tmp && "$BENCHMARK_ROOT" != /tmp/* ]]
|
||||
[[ "$BENCHMARK_ROOT" != *$'\n'* && "$BENCHMARK_ROOT" != *$'\r'* ]]
|
||||
@@ -159,15 +171,16 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
outputs:
|
||||
label: ${{ steps.ecs.outputs.label }}
|
||||
instance_id: ${{ steps.ecs.outputs.instance_id }}
|
||||
runner_name: ${{ steps.ecs.outputs.runner_name }}
|
||||
label: ${{ steps.ecs.outputs.label || steps.aws.outputs.label }}
|
||||
instance_id: ${{ steps.ecs.outputs.instance_id || steps.aws.outputs.instance_id }}
|
||||
runner_name: ${{ steps.ecs.outputs.runner_name || steps.aws.outputs.runner_name }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Create ephemeral ECS runner
|
||||
if: inputs.provider == 'Aliyun'
|
||||
id: ecs
|
||||
uses: ./greptimedb/.github/actions/aliyun-ecs-create
|
||||
with:
|
||||
@@ -178,13 +191,35 @@ jobs:
|
||||
vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }}
|
||||
security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }}
|
||||
image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }}
|
||||
instance-type: ${{ env.ECS_INSTANCE_TYPE }}
|
||||
instance-type: ${{ needs.validate.outputs.instance_type }}
|
||||
enable-docker: 'true'
|
||||
system-disk-gib: ${{ needs.validate.outputs.system_disk_gib }}
|
||||
ttl-hours: ${{ inputs.janitor_ttl_hours }}
|
||||
resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }}
|
||||
runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }}
|
||||
runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }}
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
if: inputs.provider == 'AWS'
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ vars.EC2_RUNNER_REGION }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
if: inputs.provider == 'AWS'
|
||||
- name: Create ephemeral EC2 runner
|
||||
if: inputs.provider == 'AWS'
|
||||
id: aws
|
||||
env:
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
AWS_EC2_IMAGE_ID: ${{ vars.BENCHMARK_EC2_IMAGE_ID || 'auto' }}
|
||||
AWS_EC2_INSTANCE_TYPE: ${{ needs.validate.outputs.instance_type }}
|
||||
AWS_EC2_SUBNET_ID: ${{ vars.EC2_RUNNER_SUBNET_ID }}
|
||||
AWS_EC2_SECURITY_GROUP_ID: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
|
||||
AWS_EC2_SYSTEM_DISK_GIB: ${{ needs.validate.outputs.system_disk_gib }}
|
||||
AWS_EC2_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
|
||||
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
run: uv run greptimedb/.github/scripts/aws-ec2-runner-provision.py
|
||||
benchmark:
|
||||
needs:
|
||||
- validate
|
||||
@@ -202,6 +237,12 @@ jobs:
|
||||
TSDG_BATCH_SIZE: '100000'
|
||||
O11YBENCH_GENERATOR_CPUS: '4'
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Set up benchmark dependencies
|
||||
uses: ./greptimedb/.github/actions/setup-benchmark
|
||||
- name: Checkout matching benchmark tools
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -283,7 +324,7 @@ jobs:
|
||||
needs:
|
||||
- provision
|
||||
- benchmark
|
||||
if: ${{ always() && needs.provision.outputs.instance_id != '' }}
|
||||
if: ${{ always() && (needs.provision.outputs.instance_id != '' || (inputs.provider == 'AWS' && needs.provision.result != 'skipped')) }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
@@ -292,6 +333,7 @@ jobs:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Delete ECS and unregister runner
|
||||
if: inputs.provider == 'Aliyun'
|
||||
uses: ./greptimedb/.github/actions/aliyun-ecs-delete
|
||||
with:
|
||||
access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }}
|
||||
@@ -300,3 +342,20 @@ jobs:
|
||||
region-id: ${{ vars.ALIYUN_ECS_REGION_ID }}
|
||||
instance-id: ${{ needs.provision.outputs.instance_id }}
|
||||
runner-name: ${{ needs.provision.outputs.runner_name }}
|
||||
- uses: aws-actions/configure-aws-credentials@v4
|
||||
if: inputs.provider == 'AWS'
|
||||
with:
|
||||
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
aws-region: ${{ vars.EC2_RUNNER_REGION }}
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
if: inputs.provider == 'AWS'
|
||||
- name: Delete EC2 and unregister runner
|
||||
if: inputs.provider == 'AWS'
|
||||
env:
|
||||
GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
|
||||
AWS_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
AWS_EC2_INSTANCE_ID: ${{ needs.provision.outputs.instance_id }}
|
||||
AWS_EC2_RUNNER_NAME: ${{ needs.provision.outputs.runner_name }}
|
||||
AWS_EC2_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }}
|
||||
run: uv run greptimedb/.github/scripts/aws-ec2-runner-teardown.py
|
||||
|
||||
@@ -30,6 +30,9 @@ from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
SCRIPTS_DIR = Path(__file__).parents[2] / ".github/scripts"
|
||||
sys.path.insert(0, str(SCRIPTS_DIR))
|
||||
|
||||
import runner_utils
|
||||
|
||||
|
||||
def load_module(name: str, filename: str):
|
||||
@@ -58,7 +61,7 @@ class GitHubApiResponseTest(unittest.TestCase):
|
||||
response.read.return_value = body
|
||||
response.__enter__ = Mock(return_value=response)
|
||||
response.__exit__ = Mock(return_value=False)
|
||||
with patch.object(provision.urllib.request, "urlopen", return_value=response):
|
||||
with patch.object(runner_utils.urllib.request, "urlopen", return_value=response):
|
||||
self.assertEqual(provision.github_api("token", "DELETE", "/test"), expected)
|
||||
|
||||
def test_invalid_json_still_fails(self):
|
||||
@@ -66,10 +69,26 @@ class GitHubApiResponseTest(unittest.TestCase):
|
||||
response.read.return_value = b"not json"
|
||||
response.__enter__ = Mock(return_value=response)
|
||||
response.__exit__ = Mock(return_value=False)
|
||||
with patch.object(provision.urllib.request, "urlopen", return_value=response):
|
||||
with patch.object(runner_utils.urllib.request, "urlopen", return_value=response):
|
||||
with self.assertRaises(ValueError):
|
||||
provision.github_api("token", "GET", "/test")
|
||||
|
||||
def test_http_errors_still_exit_instead_of_becoming_recoverable(self):
|
||||
error = runner_utils.urllib.error.HTTPError(
|
||||
"https://api.github.com/test", 403, "Forbidden", {}, io.BytesIO(b"denied")
|
||||
)
|
||||
with patch.object(runner_utils.urllib.request, "urlopen", side_effect=error):
|
||||
with self.assertRaisesRegex(SystemExit, "HTTP 403: denied"):
|
||||
provision.github_api("token", "GET", "/test")
|
||||
|
||||
def test_unregister_preserves_legacy_failure_semantics(self):
|
||||
with patch.object(runner_utils, "find_runner_by_name", return_value={"id": 42}):
|
||||
with patch.object(runner_utils, "github_api", side_effect=SystemExit("HTTP 403")):
|
||||
with self.assertRaisesRegex(SystemExit, "HTTP 403"):
|
||||
runner_utils.deregister_runner("token", "owner/repo", "runner")
|
||||
with patch.object(runner_utils, "github_api", side_effect=RuntimeError("connection lost")):
|
||||
self.assertFalse(runner_utils.deregister_runner("token", "owner/repo", "runner"))
|
||||
|
||||
|
||||
class ProvisionNamingTest(unittest.TestCase):
|
||||
def test_runner_name_and_label_derive_from_run_id(self) -> None:
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Offline regression coverage for CI runner selection and lifecycle."""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
SCRIPTS = Path(__file__).parents[2] / ".github/scripts"
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
def load(name, filename):
|
||||
spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
config = load("ci_config_test", "ci-runner-config.py")
|
||||
aws = load("aws_provision_test", "aws-ec2-runner-provision.py")
|
||||
teardown = load("aws_teardown_test", "aws-ec2-runner-teardown.py")
|
||||
|
||||
|
||||
def provision_config(**overrides):
|
||||
return aws.ProvisionConfig(**({
|
||||
"region": "ap-southeast-1", "image_id": "ami-test", "instance_type": "c7i.2xlarge",
|
||||
"subnet_id": "subnet-test", "security_group_id": "sg-test", "repo": "owner/repo",
|
||||
"run_id": "123-2", "github_token": "secret", "system_disk_gib": 80,
|
||||
} | overrides))
|
||||
|
||||
|
||||
def client_with_image(size=40, arch="x86_64"):
|
||||
client = Mock()
|
||||
client.describe_images.return_value = {"Images": [{
|
||||
"State": "available", "Architecture": arch, "RootDeviceName": "/dev/sda1",
|
||||
"BlockDeviceMappings": [{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": size}}],
|
||||
}]}
|
||||
client.describe_instance_types.return_value = {"InstanceTypes": [{
|
||||
"ProcessorInfo": {"SupportedArchitectures": ["x86_64"]},
|
||||
}]}
|
||||
client.run_instances.return_value = {"Instances": [{"InstanceId": "i-test"}]}
|
||||
client.get_console_output.return_value = {"Output": "boot log"}
|
||||
return client
|
||||
|
||||
|
||||
class ProviderConfigTest(unittest.TestCase):
|
||||
def test_defaults_and_overrides(self):
|
||||
for provider, requested, expected in (
|
||||
("Aliyun", "auto", "ecs.c9i.2xlarge"), ("AWS", "auto", "c7i.2xlarge"),
|
||||
("Aliyun", "ecs.c9i.4xlarge", "ecs.c9i.4xlarge"),
|
||||
("AWS", "c6i.4xlarge", "c6i.4xlarge"),
|
||||
("AWS", "c7i.metal-24xl", "c7i.metal-24xl"),
|
||||
):
|
||||
with self.subTest(provider=provider, requested=requested):
|
||||
self.assertEqual(config.resolve_instance_type(provider, requested), expected)
|
||||
|
||||
def test_wrong_provider_and_shell_fragments_rejected(self):
|
||||
for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"),
|
||||
("Aliyun", "c7i.2xlarge"), ("AWS", ""),
|
||||
("AWS", "c7i.2xlarge; touch /tmp/unexpected")):
|
||||
with self.subTest(provider=provider, requested=requested), self.assertRaises(ValueError):
|
||||
config.resolve_instance_type(provider, requested)
|
||||
|
||||
def test_cli_writes_resolved_output(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output = Path(tmp) / "output"
|
||||
result = subprocess.run([sys.executable, str(SCRIPTS / "ci-runner-config.py"),
|
||||
"--provider", "AWS"],
|
||||
env=os.environ | {"GITHUB_OUTPUT": str(output)},
|
||||
capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(output.read_text(), "instance_type=c7i.2xlarge\n")
|
||||
|
||||
|
||||
|
||||
class BenchmarkSetupTest(unittest.TestCase):
|
||||
def run_setup(self, missing_tools="", docker_installed=True, docker_ready=True):
|
||||
action = (SCRIPTS.parent / "actions/setup-benchmark/action.yml").read_text()
|
||||
steps = re.findall(r" run: \|\n((?: .*\n|\n)+)", action)
|
||||
self.assertEqual(len(steps), 2)
|
||||
script = "\n".join(textwrap.dedent(step) for step in steps)
|
||||
# Execute the actual action twice with host commands stubbed: installation
|
||||
# changes availability, so the second pass must perform no mutations.
|
||||
prelude = r"""
|
||||
function . { ID=ubuntu; VERSION_CODENAME=noble; }
|
||||
uname() { echo x86_64; }
|
||||
dpkg() { echo amd64; }
|
||||
curl() { echo fixture-key; }
|
||||
command() {
|
||||
if [[ "$1" == -v ]]; then
|
||||
if [[ "$2" == docker ]]; then [[ "$DOCKER_INSTALLED" == true ]]; return; fi
|
||||
if [[ " $MISSING_TOOLS " == *" $2 "* ]]; then return 1; fi
|
||||
return 0
|
||||
fi
|
||||
builtin command "$@"
|
||||
}
|
||||
docker() { [[ "$DOCKER_READY" == true ]]; }
|
||||
sudo() {
|
||||
printf '%s\n' "$*" >> "$CALLS"
|
||||
if [[ "$*" == *' apt-get install '* ]]; then
|
||||
MISSING_TOOLS=''
|
||||
[[ "$*" != *' docker-ce '* ]] || DOCKER_INSTALLED=true
|
||||
fi
|
||||
if [[ "$*" == *'systemctl enable --now docker' ]]; then DOCKER_READY=true; fi
|
||||
if [[ "$*" == *' tee '* ]]; then while IFS= read -r line; do :; done; fi
|
||||
return 0
|
||||
}
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
calls = Path(tmp) / "calls"
|
||||
calls.touch()
|
||||
env = os.environ | {"CALLS": str(calls), "MISSING_TOOLS": missing_tools,
|
||||
"DOCKER_INSTALLED": str(docker_installed).lower(),
|
||||
"DOCKER_READY": str(docker_ready).lower()}
|
||||
result = subprocess.run(["bash", "-c", prelude + script +
|
||||
'\nprintf "SECOND PASS\\n" >> "$CALLS"\n' + script],
|
||||
env=env, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
first, second = calls.read_text().split("SECOND PASS\n")
|
||||
self.assertEqual(second, "", "Second setup must reuse installed dependencies")
|
||||
return first
|
||||
|
||||
def test_prepared_runner_skips_all_privileged_changes(self):
|
||||
self.assertEqual(self.run_setup(), "")
|
||||
|
||||
def test_missing_tools_are_installed_without_changing_docker(self):
|
||||
calls = self.run_setup(missing_tools="jq rsync")
|
||||
self.assertIn("--no-install-recommends jq rsync", calls)
|
||||
self.assertIn("NEEDRESTART_MODE=l", calls)
|
||||
self.assertNotIn("docker", calls)
|
||||
|
||||
def test_missing_docker_is_installed_once(self):
|
||||
calls = self.run_setup(docker_installed=False, docker_ready=False)
|
||||
self.assertIn("NEEDRESTART_MODE=l", calls)
|
||||
self.assertIn("docker-ce docker-ce-cli containerd.io docker-buildx-plugin", calls)
|
||||
self.assertIn("systemctl enable --now docker", calls)
|
||||
|
||||
def test_existing_stopped_docker_is_started_without_installation(self):
|
||||
self.assertEqual(self.run_setup(docker_ready=False), "-n systemctl enable --now docker\n")
|
||||
|
||||
|
||||
class AwsProvisionTest(unittest.TestCase):
|
||||
def test_host_policy_matches_aliyun_before_runner_activation(self):
|
||||
aliyun = load("aliyun_host_policy_test", "aliyun-ecs-runner-provision.py")
|
||||
ecs_script = aliyun.render_user_data("name", "label", "token", "owner/repo")
|
||||
aws_script = aws.render_user_data("name", "token", "owner/repo", "https://example.com/runner.tgz")
|
||||
start = ecs_script.index("# Mask systemd-oomd")
|
||||
end = ecs_script.index("free --human", start) + len("free --human")
|
||||
policy = ecs_script[start:end]
|
||||
self.assertIn(policy, aws_script)
|
||||
self.assertLess(aws_script.index(policy), aws_script.index("./svc.sh start"))
|
||||
self.assertLess(aws_script.index("OOMPolicy=continue"), aws_script.index("./svc.sh start"))
|
||||
self.assertLess(aws_script.index("./svc.sh install runner"), aws_script.index("service_name=$(cat .service)"))
|
||||
self.assertIn('"/etc/systemd/system/$service_name.d/oom.conf"', aws_script)
|
||||
|
||||
def test_ubuntu_image_resolution_and_override(self):
|
||||
client = Mock()
|
||||
client.describe_images.return_value = {"Images": [
|
||||
{"ImageId": "ami-new", "CreationDate": "2026-09-04", "Name": "noble-new"},
|
||||
{"ImageId": "ami-old", "CreationDate": "2026-08-04", "Name": "noble-old"},
|
||||
]}
|
||||
self.assertEqual(aws.resolve_image(client, "auto"), "ami-new")
|
||||
request = client.describe_images.call_args.kwargs
|
||||
self.assertEqual(request["Owners"], ["099720109477"])
|
||||
self.assertIn({"Name": "name", "Values": [
|
||||
"ubuntu/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64-server-*"]}, request["Filters"])
|
||||
client.reset_mock()
|
||||
self.assertEqual(aws.resolve_image(client, "ami-pinned"), "ami-pinned")
|
||||
client.describe_images.assert_not_called()
|
||||
client.describe_images.return_value = {"Images": []}
|
||||
with self.assertRaisesRegex(ValueError, "No Canonical Ubuntu"):
|
||||
aws.resolve_image(client, "auto")
|
||||
|
||||
def test_disk_size_and_architecture_fail_before_allocation(self):
|
||||
for client, pattern in ((client_with_image(size=500), "smaller than AMI"),
|
||||
(client_with_image(arch="arm64"), "x86_64")):
|
||||
with self.subTest(pattern=pattern), patch.object(aws, "create_registration_token") as token:
|
||||
with self.assertRaisesRegex(ValueError, pattern):
|
||||
aws.run_instance(client, provision_config())
|
||||
client.run_instances.assert_not_called()
|
||||
token.assert_not_called()
|
||||
|
||||
def test_arm_instance_rejected_before_registration(self):
|
||||
client = client_with_image()
|
||||
client.describe_instance_types.return_value["InstanceTypes"][0]["ProcessorInfo"]["SupportedArchitectures"] = ["arm64"]
|
||||
with patch.object(aws, "create_registration_token") as token:
|
||||
with self.assertRaisesRegex(ValueError, "x86_64"):
|
||||
aws.run_instance(client, provision_config(instance_type="r8g.2xlarge"))
|
||||
token.assert_not_called()
|
||||
client.run_instances.assert_not_called()
|
||||
|
||||
def test_create_request_and_recovery_tags(self):
|
||||
client = client_with_image()
|
||||
with patch.object(aws, "create_registration_token", return_value="short-lived-token"), \
|
||||
patch.object(aws, "runner_download", return_value="https://example.com/runner.tar.gz"):
|
||||
self.assertEqual(aws.run_instance(client, provision_config(system_disk_gib=100)), "i-test")
|
||||
request = client.run_instances.call_args.kwargs
|
||||
self.assertEqual(request["BlockDeviceMappings"], [{"DeviceName": "/dev/sda1", "Ebs": {
|
||||
"VolumeSize": 100, "VolumeType": "gp3", "Iops": 3000, "Throughput": 125,
|
||||
"DeleteOnTermination": True}}])
|
||||
self.assertEqual(request["ClientToken"], "ci-ec2-123-2")
|
||||
self.assertNotIn("InstanceMarketOptions", request)
|
||||
self.assertNotIn("IamInstanceProfile", request)
|
||||
for specification in request["TagSpecifications"]:
|
||||
tags = {t["Key"]: t["Value"] for t in specification["Tags"]}
|
||||
self.assertEqual(tags["managed-by"], "query-regression-ci")
|
||||
self.assertEqual(tags[aws.REPO_TAG], "owner/repo")
|
||||
self.assertEqual(tags["query-regression-run-id"], "123-2")
|
||||
self.assertEqual(tags["runner-ttl-hours"], "8")
|
||||
self.assertNotIn("secret", request["UserData"])
|
||||
self.assertIn("--token short-lived-token", request["UserData"])
|
||||
self.assertNotIn("apt-get install", request["UserData"])
|
||||
self.assertIn('for tool in git curl tar sudo; do command -v "$tool"; done', request["UserData"])
|
||||
self.assertLess(request["UserData"].index("usermod -aG docker runner"),
|
||||
request["UserData"].index("./svc.sh start"))
|
||||
self.assertIn("visudo -cf /etc/sudoers.d/ci-runner", request["UserData"])
|
||||
self.assertLess(request["UserData"].index("$nrconf{override_rc}"),
|
||||
request["UserData"].index("./svc.sh start"))
|
||||
check = subprocess.run(["bash", "-n"], input=request["UserData"], text=True, capture_output=True)
|
||||
self.assertEqual(check.returncode, 0, check.stderr)
|
||||
|
||||
def test_partial_provision_keeps_outputs_and_console(self):
|
||||
client = client_with_image()
|
||||
client.get_waiter.return_value.wait.side_effect = TimeoutError("not running")
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
output = Path(tmp) / "output"
|
||||
with patch.dict(os.environ, {"GITHUB_OUTPUT": str(output)}), patch.object(aws, "run_instance", return_value="i-test"):
|
||||
with self.assertRaises(TimeoutError):
|
||||
aws.provision(client, provision_config())
|
||||
self.assertEqual(output.read_text().splitlines(), [
|
||||
"instance_id=i-test", "label=ci-ec2-123-2",
|
||||
"runner_name=ci-ec2-123-2"])
|
||||
client.get_console_output.assert_called_once()
|
||||
|
||||
def test_online_poll_and_timeout(self):
|
||||
for online in (True, False):
|
||||
with self.subTest(online=online), patch.object(aws, "run_instance", return_value="i-test"), \
|
||||
patch.object(aws, "find_runner_by_name", side_effect=[None, {"status": "online"}]), \
|
||||
patch.object(aws.time, "sleep"), \
|
||||
patch.object(aws.time, "monotonic", side_effect=[0, 1, 2 if online else 601]):
|
||||
client = client_with_image()
|
||||
if online:
|
||||
self.assertEqual(aws.provision(client, provision_config()), 0)
|
||||
else:
|
||||
with self.assertRaises(TimeoutError):
|
||||
aws.provision(client, provision_config())
|
||||
client.get_console_output.assert_called_once()
|
||||
|
||||
|
||||
class AwsTeardownTest(unittest.TestCase):
|
||||
NOW = datetime(2026, 9, 23, 10, tzinfo=timezone.utc)
|
||||
|
||||
def setUp(self):
|
||||
sleeper = patch.object(teardown.time, "sleep")
|
||||
sleeper.start()
|
||||
self.addCleanup(sleeper.stop)
|
||||
|
||||
def instance(self, identity="i-test", age=9, ttl="8"):
|
||||
tags = [{"Key": teardown.RUNNER_TAG, "Value": "ci-ec2-123-2"}]
|
||||
if ttl is not None:
|
||||
tags.append({"Key": teardown.TTL_TAG_KEY, "Value": ttl})
|
||||
return {"InstanceId": identity, "LaunchTime": self.NOW - timedelta(hours=age), "Tags": tags}
|
||||
|
||||
def test_pagination_and_ownership_filters(self):
|
||||
client = Mock()
|
||||
client.get_paginator.return_value.paginate.return_value = [
|
||||
{"Reservations": [{"Instances": [self.instance("i-a")]}]},
|
||||
{"Reservations": [{"Instances": [self.instance("i-b")]}]},
|
||||
]
|
||||
self.assertEqual([i["InstanceId"] for i in teardown.managed_instances(client, "owner/repo", "123-2")], ["i-a", "i-b"])
|
||||
self.assertEqual(client.get_paginator.return_value.paginate.call_args.kwargs["Filters"], [
|
||||
{"Name": "tag:managed-by", "Values": ["query-regression-ci"]},
|
||||
{"Name": "tag:github-repository", "Values": ["owner/repo"]},
|
||||
{"Name": "tag:query-regression-run-id", "Values": ["123-2"]},
|
||||
])
|
||||
|
||||
def test_sweep_expiry_and_missing_or_invalid_ttl(self):
|
||||
instances = [self.instance("expired", age=8), self.instance("fresh", age=7),
|
||||
self.instance("invalid", ttl="bad"),
|
||||
self.instance("missing-expired", age=4, ttl=None),
|
||||
self.instance("missing-fresh", age=3, ttl=None)]
|
||||
with patch.object(teardown, "managed_instances", return_value=instances), \
|
||||
patch.object(teardown, "delete_instance", return_value=True) as delete, \
|
||||
patch.object(teardown, "deregister_runner", return_value=True) as unregister:
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", now=self.NOW), 0)
|
||||
self.assertEqual([c.args[1] for c in delete.call_args_list], ["expired", "missing-expired"])
|
||||
self.assertEqual(unregister.call_count, 2)
|
||||
|
||||
def test_ttl_decisions_match_aliyun(self):
|
||||
aliyun = load("aliyun_ttl_policy_test", "aliyun-ecs-runner-teardown.py")
|
||||
for age, ttl in [(3, None), (4, None), (7, "8"), (8, "8"), (9, "8"),
|
||||
(10, "bad"), (10, ""), (10, "0"), (200, "169"), (168, "168")]:
|
||||
instance = self.instance(age=age, ttl=ttl)
|
||||
created = instance["LaunchTime"].strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
expected = aliyun.expired_instance_names(
|
||||
[(instance["InstanceId"], "runner", created, ttl)], self.NOW, timedelta(hours=4))
|
||||
with self.subTest(age=age, ttl=ttl), \
|
||||
patch.object(teardown, "managed_instances", return_value=[instance]), \
|
||||
patch.object(teardown, "delete_instance", return_value=True) as delete, \
|
||||
patch.object(teardown, "deregister_runner", return_value=True):
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", now=self.NOW), 0)
|
||||
self.assertEqual([call.args[1] for call in delete.call_args_list],
|
||||
[identity for identity, _ in expected])
|
||||
|
||||
def test_targeted_cleanup_recovers_missing_outputs_without_waiting_for_ttl(self):
|
||||
with patch.object(teardown, "managed_instances", return_value=[self.instance(age=0)]), \
|
||||
patch.object(teardown, "delete_instance", return_value=True) as delete, \
|
||||
patch.object(teardown, "deregister_runner", return_value=True) as unregister:
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 0)
|
||||
delete.assert_called_once()
|
||||
unregister.assert_called_once_with("token", "owner/repo", "ci-ec2-123-2")
|
||||
|
||||
def test_independent_cloud_and_github_cleanup_failures(self):
|
||||
for cloud_ok, github_result in ((False, True), (True, RuntimeError("GitHub unavailable"))):
|
||||
with self.subTest(cloud_ok=cloud_ok), \
|
||||
patch.object(teardown, "managed_instances", return_value=[self.instance()]), \
|
||||
patch.object(teardown, "delete_instance", return_value=cloud_ok) as delete, \
|
||||
patch.object(teardown, "deregister_runner", side_effect=[github_result]) as unregister:
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1)
|
||||
delete.assert_called_once()
|
||||
unregister.assert_called_once()
|
||||
|
||||
def test_already_gone_instance_still_unregisters(self):
|
||||
with patch.object(teardown, "managed_instances", return_value=[]), \
|
||||
patch.object(teardown, "deregister_runner", return_value=True) as unregister:
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 0)
|
||||
unregister.assert_called_once()
|
||||
|
||||
def test_retried_teardown_uses_original_instance_and_runner(self):
|
||||
client = Mock()
|
||||
client.get_paginator.return_value.paginate.return_value = [
|
||||
{"Reservations": [{"Instances": [self.instance()]}]},
|
||||
]
|
||||
with patch.object(teardown, "delete_instance", return_value=True) as delete, \
|
||||
patch.object(teardown, "deregister_runner", return_value=True) as unregister:
|
||||
self.assertEqual(teardown.teardown(client, "owner/repo", "token", run_id="123-3",
|
||||
instance_id="i-test", runner_name="ci-ec2-123-2"), 0)
|
||||
filters = client.get_paginator.return_value.paginate.call_args.kwargs["Filters"]
|
||||
self.assertIn({"Name": "instance-id", "Values": ["i-test"]}, filters)
|
||||
self.assertFalse(any(f["Name"] == "tag:query-regression-run-id" for f in filters))
|
||||
delete.assert_called_once_with(client, "i-test")
|
||||
unregister.assert_called_once_with("token", "owner/repo", "ci-ec2-123-2")
|
||||
client.get_paginator.return_value.paginate.return_value = []
|
||||
with patch.object(teardown, "deregister_runner", return_value=True) as unregister:
|
||||
self.assertEqual(teardown.teardown(client, "owner/repo", "token", run_id="123-3",
|
||||
instance_id="i-test", runner_name="ci-ec2-123-2"), 0)
|
||||
unregister.assert_called_once_with("token", "owner/repo", "ci-ec2-123-2")
|
||||
|
||||
def test_terminate_waits_and_handles_not_found(self):
|
||||
client = client_with_image()
|
||||
self.assertTrue(teardown.delete_instance(client, "i-test"))
|
||||
client.get_waiter.assert_called_with("instance_terminated")
|
||||
error = RuntimeError("gone")
|
||||
error.response = {"Error": {"Code": "InvalidInstanceID.NotFound"}}
|
||||
client.terminate_instances.side_effect = error
|
||||
with patch.object(teardown.time, "sleep"):
|
||||
self.assertTrue(teardown.delete_instance(client, "i-test"))
|
||||
|
||||
def test_transient_not_found_is_retried(self):
|
||||
client = client_with_image()
|
||||
error = RuntimeError("not visible yet")
|
||||
error.response = {"Error": {"Code": "InvalidInstanceID.NotFound"}}
|
||||
client.terminate_instances.side_effect = [error, {}]
|
||||
self.assertTrue(teardown.delete_instance(client, "i-test"))
|
||||
self.assertEqual(client.terminate_instances.call_count, 2)
|
||||
|
||||
def test_missing_tags_are_polled_before_cleanup(self):
|
||||
with patch.object(teardown, "managed_instances", side_effect=[[], [self.instance()]]), \
|
||||
patch.object(teardown, "delete_instance", return_value=True) as delete, \
|
||||
patch.object(teardown, "deregister_runner", return_value=True):
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 0)
|
||||
delete.assert_called_once()
|
||||
|
||||
def test_github_system_exit_does_not_skip_other_instances(self):
|
||||
with patch.object(teardown, "managed_instances", return_value=[self.instance("i-a"), self.instance("i-b")]), \
|
||||
patch.object(teardown, "delete_instance", return_value=True) as delete, \
|
||||
patch.object(teardown, "deregister_runner", side_effect=[SystemExit("HTTP 403"), True]):
|
||||
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1)
|
||||
self.assertEqual([c.args[1] for c in delete.call_args_list], ["i-a", "i-b"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user