diff --git a/.github/runner-scale-sets/query-regression/README.md b/.github/runner-scale-sets/query-regression/README.md index 77b7d604f1..98fd7061fc 100644 --- a/.github/runner-scale-sets/query-regression/README.md +++ b/.github/runner-scale-sets/query-regression/README.md @@ -90,9 +90,15 @@ tool contract. Build a new ECS image from it: ALIBABA_CLOUD_ACCESS_KEY_ID=... ALIBABA_CLOUD_ACCESS_KEY_SECRET=... \ uv run .github/runner-scale-sets/query-regression/ecs-image/build-ecs-image.py \ --region-id --vswitch-id --security-group-id \ - --base-image-id + --base-image-id ``` +The configured image has passed the workflow host checks for mold `2.40.4` +and Python `3.14.4`. The builder installs those host tools from distribution +packages; it does not pin or copy them from the container. A stock Ubuntu 24.04 +base does not supply these versions. Validate the chosen base and the resulting +host against the workflow checks before replacing the configured image. + The script boots a temporary builder instance, `docker build`s the runner image, materializes `/opt/rustup`, `/opt/cargo`, `/usr/local/bin` tools, and `/home/runner` (actions-runner) onto the host, installs the ephemeral-runner diff --git a/.github/scripts/aliyun-ecs-runner-provision.py b/.github/scripts/aliyun-ecs-runner-provision.py index 307ee336eb..406c0f51c9 100644 --- a/.github/scripts/aliyun-ecs-runner-provision.py +++ b/.github/scripts/aliyun-ecs-runner-provision.py @@ -216,13 +216,15 @@ def github_api(token: str, method: str, path: str, body: dict | None = None) -> ) try: with urllib.request.urlopen(request, timeout=30) as response: - return json.loads(response.read().decode("utf-8")) + payload = response.read() + return json.loads(payload.decode("utf-8")) if payload else {} except urllib.error.HTTPError as error: # GitHub's error body says exactly why (e.g. "Must have admin rights to # Repository" for a PAT without the required scope); surface it instead - # of a bare "HTTP Error 403". + # of a bare "HTTP Error 403". This must remain catchable by teardown so + # a runner deregistration failure does not prevent ECS cleanup. body = error.read().decode("utf-8", "replace") - raise SystemExit( + raise RuntimeError( f"GitHub API {method} {path} failed: HTTP {error.code}: {body}\n" "The token comes from the GH_PERSONAL_ACCESS_TOKEN secret; it needs " "'repo' scope (classic PAT) or 'Administration: write' on the " diff --git a/.github/scripts/aliyun-ecs-runner-teardown.py b/.github/scripts/aliyun-ecs-runner-teardown.py index 36c80253b9..f47821f1f7 100644 --- a/.github/scripts/aliyun-ecs-runner-teardown.py +++ b/.github/scripts/aliyun-ecs-runner-teardown.py @@ -144,11 +144,11 @@ def delete_instance(client, instance_id: str, region_id: str | None = None) -> b def deregister_runner(token: str, repo: str, runner_name: str) -> bool: - runner = provision.find_runner_by_name(token, repo, runner_name) - if runner is None: - print(f"Runner {runner_name} is not registered", flush=True) - return True try: + runner = provision.find_runner_by_name(token, repo, runner_name) + if runner is None: + print(f"Runner {runner_name} is not registered", flush=True) + return True provision.github_api(token, "DELETE", f"/repos/{repo}/actions/runners/{runner['id']}") print(f"Deregistered runner {runner_name} (id {runner['id']})", flush=True) return True diff --git a/tests/perf/AGENTS.md b/tests/perf/AGENTS.md index 45027eb1d0..8b93a20c3d 100644 --- a/tests/perf/AGENTS.md +++ b/tests/perf/AGENTS.md @@ -33,9 +33,10 @@ - Keep the direct-SST generator generic. Issue-specific behavior belongs in case files and thresholds, not in Rust generator logic. - Before pushing perf harness changes, run at least: - - `uv run --no-project python -m py_compile .github/scripts/query-regression-run.py .github/scripts/query-regression-summary.py .github/scripts/query-regression-pr-metadata.py tests/perf/test_query_regression_runner_compaction_toctou.py tests/perf/test_query_regression_runner_otlp_trace_load.py` - - `uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py && uv run --no-project python tests/perf/test_query_regression_runner_otlp_trace_load.py` + - the six Python tooling tests listed in the `test-tooling` job of + `.github/workflows/query-regression.yml` (also run by `develop.yml`). - `cargo fmt --all -- --check` - `cargo build -p cmd --bin query_perf_fixture --features dev-tools` + - `cargo build -p cmd --bin query_regression_runner --features dev-tools` - exercise the outer lifecycle script and Rust fixture generator against all built-in cases when the DSL or workflow case selection changes. diff --git a/tests/perf/README.md b/tests/perf/README.md index a8869204ad..560de4ca01 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -244,8 +244,9 @@ improvement. The case passes when every `actual_pct` is at or below its the case at least three times on an otherwise idle machine and compare the median regressions rather than relying on one run. -The CI runner image includes the pinned `otelgen` binary. Until this case is -added to the default set, run it explicitly with `workflow_dispatch`: +The default `aliyun-ecs` runner provisions a fresh ECS instance from the +query-regression image, which includes the pinned `otelgen` binary. Until this +case is added to the default set, run it explicitly with `workflow_dispatch`: ```bash gh workflow run query-regression.yml \ @@ -255,12 +256,9 @@ gh workflow run query-regression.yml \ -f candidate_ref= \ -f cargo_profile=nightly \ -f http_timeout=300 \ - -f runner=perf-regression-8-cores + -f runner=aliyun-ecs ``` -The selected ARC scale set must already be deployed with the runner-image -digest built from the current query-regression Dockerfile. - ## Generator contract The direct-SST generator should accept a case definition with: diff --git a/tests/perf/test_aliyun_ecs_runner_scripts.py b/tests/perf/test_aliyun_ecs_runner_scripts.py index 581fc88abd..0cfe34cc4b 100644 --- a/tests/perf/test_aliyun_ecs_runner_scripts.py +++ b/tests/perf/test_aliyun_ecs_runner_scripts.py @@ -21,6 +21,7 @@ import sys import unittest from datetime import datetime, timedelta, timezone from pathlib import Path +from unittest import mock SCRIPTS_DIR = Path(__file__).parents[2] / ".github/scripts" @@ -45,6 +46,25 @@ class ProvisionNamingTest(unittest.TestCase): self.assertEqual(provision.runner_label_for_run("12345"), "query-regression-ecs-12345") +class GithubApiTest(unittest.TestCase): + def test_empty_204_response_returns_empty_object(self) -> None: + response = mock.MagicMock() + response.status = 204 + response.read.return_value = b"" + response.__enter__.return_value = response + with mock.patch.object(provision.urllib.request, "urlopen", return_value=response): + self.assertEqual(provision.github_api("TOKEN", "DELETE", "/test"), {}) + + def test_populated_json_response_is_unchanged(self) -> None: + response = mock.MagicMock() + response.read.return_value = b'{"token":"TOKEN"}' + response.__enter__.return_value = response + with mock.patch.object(provision.urllib.request, "urlopen", return_value=response): + self.assertEqual( + provision.github_api("TOKEN", "POST", "/test", body={}), {"token": "TOKEN"} + ) + + class ProvisionUserDataTest(unittest.TestCase): def render(self) -> str: return provision.render_user_data( @@ -90,6 +110,41 @@ class ProvisionUserDataTest(unittest.TestCase): self.assertLess(script.index("swapon"), script.index("systemctl restart --no-block ephemeral-github-runner.service")) +class TeardownSweepTest(unittest.TestCase): + def test_sweep_deletes_all_instances_when_first_runner_lookup_fails(self) -> None: + client = mock.Mock() + instances = [ + ("i-first", "qreg-ecs-first", "2026-08-17T01:00Z"), + ("i-second", "qreg-ecs-second", "2026-08-17T01:00Z"), + ] + with ( + mock.patch.object(teardown, "list_managed_instances", return_value=instances), + mock.patch.object( + teardown, + "expired_instance_names", + return_value=[("i-first", "qreg-ecs-first"), ("i-second", "qreg-ecs-second")], + ), + mock.patch.object(teardown, "delete_instance", return_value=True) as delete_instance, + mock.patch.object( + teardown.provision, + "find_runner_by_name", + side_effect=[RuntimeError("GitHub unavailable"), None], + ), + ): + self.assertEqual( + teardown.sweep(client, "cn-test", "GreptimeTeam/greptimedb", "TOKEN", timedelta(hours=4)), + 1, + ) + + self.assertEqual( + delete_instance.call_args_list, + [ + mock.call(client, "i-first", "cn-test"), + mock.call(client, "i-second", "cn-test"), + ], + ) + + class TeardownExpiryTest(unittest.TestCase): NOW = datetime(2026, 8, 17, 6, 0, tzinfo=timezone.utc) TTL = timedelta(hours=4)