diff --git a/.github/actions/github-script-tests/action.yml b/.github/actions/github-script-tests/action.yml index 5721aead096..2d2a310a9ef 100644 --- a/.github/actions/github-script-tests/action.yml +++ b/.github/actions/github-script-tests/action.yml @@ -44,6 +44,7 @@ runs: python3 tests/perf/test_query_regression_runner_otlp_trace_load.py python3 tests/perf/test_query_regression_summary_otlp.py python3 tests/perf/test_query_regression_case_selection.py + python3 tests/perf/test_query_regression_helpers.py python3 tests/perf/test_query_regression_nightly_refs.py python3 tests/perf/test_query_regression_slash.py python3 tests/perf/test_aliyun_ecs_runner_scripts.py diff --git a/.github/scripts/build-query-regression-helpers.sh b/.github/scripts/build-query-regression-helpers.sh new file mode 100644 index 00000000000..6b6cdb6fcde --- /dev/null +++ b/.github/scripts/build-query-regression-helpers.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${CARGO_TARGET_DIR:?CARGO_TARGET_DIR must be configured by the runner}" +revision="$(git rev-parse HEAD)" +tag="query-regression-helpers:${revision}" +staging="$(mktemp -d)" +trap 'rm -rf -- "$staging"' EXIT +cargo build --profile nightly -p cmd --features dev-tools \ + --bin query_perf_fixture --bin query_regression_runner +for binary in query_perf_fixture query_regression_runner; do + "$CARGO_TARGET_DIR/nightly/$binary" --help >/dev/null + cp "$CARGO_TARGET_DIR/nightly/$binary" "$staging/$binary" +done +mkdir -p helpers-image helpers-validation +docker build --provenance=false --build-arg "SOURCE_REVISION=$revision" \ + --file docker/query-regression-helpers/Dockerfile --tag "$tag" "$staging" +container="$(docker create "$tag" /query_perf_fixture --help)" +trap 'docker rm --force --volumes "$container"; rm -rf -- "$staging"' EXIT +mkdir -p "$staging/extracted" +for binary in query_perf_fixture query_regression_runner; do + docker cp "$container:/$binary" "$staging/extracted/$binary" + chmod +x "$staging/extracted/$binary" + "$staging/extracted/$binary" --help >/dev/null + cmp "$staging/$binary" "$staging/extracted/$binary" +done +# Hold the revision and cases constant: compare source-built versus extracted +# fixture normalization twice. Keep the raw outputs as workflow artifacts. +printf '%s\n' "$revision" > helpers-validation/source-revision.txt +for iteration in 1 2; do + while IFS= read -r case; do + name="$(basename "$(dirname "$case")")" + for mode in source image; do + binary="$staging/query_perf_fixture" + [[ "$mode" != image ]] || binary="$staging/extracted/query_perf_fixture" + "$binary" plan --case "$case" > "helpers-validation/${name}-${iteration}-${mode}.json" + done + cmp "helpers-validation/${name}-${iteration}-source.json" \ + "helpers-validation/${name}-${iteration}-image.json" + done < <(find tests/perf/query_cases -name case.toml -type f | sort) +done +(cd "$staging" && sha256sum query_perf_fixture query_regression_runner) > helpers-image/SHA256SUMS +printf '%s\n' "$revision" > helpers-image/source-revision.txt +docker save "$tag" --output helpers-image/image.tar diff --git a/.github/scripts/publish-query-regression-helpers.sh b/.github/scripts/publish-query-regression-helpers.sh new file mode 100644 index 00000000000..b867455f498 --- /dev/null +++ b/.github/scripts/publish-query-regression-helpers.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -euo pipefail +repository=greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/query-regression-helpers +revision="$(cat helpers-image/source-revision.txt)" +[[ "$revision" == "$GITHUB_SHA" ]] +tag="${repository}:${revision}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" +# Match release-dev-builder-images.yaml: publish ACR images with Skopeo from +# a hosted runner. A failed publish can reuse the completed build artifact. +mkdir -p helpers-validation +docker run --rm \ + -v "${DOCKER_CONFIG:-$HOME/.docker}:/root/.docker:ro" \ + -v "$PWD/helpers-image:/image:ro" \ + -v "$PWD/helpers-validation:/result" \ + -e REGISTRY_AUTH_FILE=/root/.docker/config.json \ + quay.io/skopeo/stable:latest --command-timeout 20m copy --retry-times 3 \ + --digestfile /result/digest.txt \ + docker-archive:/image/image.tar "docker://$tag" +digest="$(cat helpers-validation/digest.txt)" +[[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] +image="$repository@$digest" +# The ECS build tested these exact bytes. Verify them again from the registry; +# hosted runners need not share the ECS runtime library versions. +python3 .github/scripts/query-regression-helpers.py "$image" helpers-validation/extracted \ + --checksums helpers-image/SHA256SUMS +jq -e --arg revision "$revision" '.source_revision == $revision' \ + helpers-validation/extracted/helpers-image.json >/dev/null +printf 'image=%s\n' "$image" >> "$GITHUB_OUTPUT" +printf 'Published helpers: `%s` (source `%s`)\n' "$image" "$revision" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/scripts/query-regression-helpers.py b/.github/scripts/query-regression-helpers.py new file mode 100644 index 00000000000..7f1e22a5c83 --- /dev/null +++ b/.github/scripts/query-regression-helpers.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Extract digest-pinned query regression tools from an artifact image.""" + +import argparse +import hashlib +import json +import os +import re +import subprocess +from pathlib import Path + +BINARIES = ("query_perf_fixture", "query_regression_runner") + + +def resolve_image(value): + if value == "pinned": + value = os.environ.get("QUERY_REGRESSION_HELPERS_IMAGE", "") + if not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9._/:@-]*@sha256:[0-9a-f]{64}", value): + raise ValueError("helpers_image requires an image@sha256 digest; set QUERY_REGRESSION_HELPERS_IMAGE by publishing a verified image before using 'pinned'") + return value + + +def run(*args): + return subprocess.check_output(args, text=True).strip() + + +def extract(image, destination, checksums=None): + expected = None + if checksums is not None: + entries = [line.split() for line in checksums.read_text().splitlines()] + expected = {name: digest for digest, name in entries} + if set(expected) != set(BINARIES) or len(entries) != len(BINARIES): + raise ValueError("Checksums must contain exactly the two helper binaries") + image = resolve_image(image) + destination.mkdir(parents=True, exist_ok=True) + run("docker", "pull", image) + metadata = json.loads(run("docker", "image", "inspect", image))[0] + revision = (metadata.get("Config", {}).get("Labels") or {}).get("org.opencontainers.image.revision", "") + if not re.fullmatch(r"[0-9a-f]{40}", revision): + raise ValueError("Helper image is missing its source revision label") + container = run("docker", "create", image, "/query_perf_fixture", "--help") + try: + for binary in BINARIES: + target = destination / binary + run("docker", "cp", f"{container}:/{binary}", str(target)) + target.chmod(0o755) + if expected is None: + run(str(target.resolve()), "--help") + else: + with target.open("rb") as file: + actual = hashlib.file_digest(file, "sha256").hexdigest() + if actual != expected[binary]: + raise ValueError(f"Published helper checksum mismatch: {binary}") + finally: + run("docker", "rm", "--force", "--volumes", container) + provenance = {"image": image, "image_id": metadata["Id"], "source_revision": revision} + (destination / "helpers-image.json").write_text(json.dumps(provenance, indent=2) + "\n") + print(json.dumps(provenance)) + return provenance + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("image") + parser.add_argument("destination", type=Path) + parser.add_argument("--checksums", type=Path, help="Verify build checksums instead of executing on this host") + args = parser.parse_args() + extract(args.image, args.destination, args.checksums) diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index d488cee1594..e59dcdfb709 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -21,10 +21,13 @@ import argparse import json import os import re +import shlex import signal import socket import subprocess +import sys import time +import uuid import urllib.request from dataclasses import dataclass from pathlib import Path @@ -130,6 +133,7 @@ class RunTarget: datanode_http_port: int datanode_data_dir: Path frontend_config: Path | None = None + containerized: bool = False def allocate_ports(n: int) -> list[int]: @@ -151,6 +155,7 @@ def make_target( root: Path, ports: list[int], frontend_config: Path | None = None, + containerized: bool = False, ) -> RunTarget: work_dir = root / name return RunTarget( @@ -167,6 +172,7 @@ def make_target( datanode_http_port=ports[3], datanode_data_dir=work_dir / "datanode-0" / "data", frontend_config=frontend_config, + containerized=containerized, ) @@ -258,7 +264,11 @@ def stop_component( procs: dict[tuple[str, str], subprocess.Popen[bytes]], ) -> None: proc = procs.pop((target.name, name), None) - if proc is None or proc.poll() is not None: + if proc is None: + return + if proc.poll() is not None: + if target.containerized and proc.returncode != 0: + raise RuntimeError(f"Docker target {target.name}/{name} exited with {proc.returncode}") return try: os.killpg(proc.pid, signal.SIGTERM) @@ -272,6 +282,8 @@ def stop_component( except ProcessLookupError: return proc.wait(timeout=20) + if target.containerized and proc.returncode not in (0, 128 + signal.SIGTERM): + raise RuntimeError(f"Docker target {target.name}/{name} cleanup failed: {proc.returncode}") def restart_component( @@ -284,9 +296,15 @@ def restart_component( def stop_all(targets: list[RunTarget], procs: dict[tuple[str, str], subprocess.Popen[bytes]]) -> None: + errors = [] for target in reversed(targets): for name in ("frontend", "datanode", "metasrv"): - stop_component(target, name, procs) + try: + stop_component(target, name, procs) + except Exception as err: + errors.append(err) + if errors: + raise RuntimeError(f"component cleanup failed: {errors}") from errors[0] def load_plan(fixture_generator: Path, case_path: Path) -> dict[str, Any]: @@ -313,8 +331,8 @@ def run_direct_case( ) -> int: ports = allocate_ports(16) targets = [ - make_target("base", base_bin, work_dir, ports[:8]), - make_target("candidate", candidate_bin, work_dir, ports[8:]), + make_target("base", base_bin, work_dir, ports[:8], containerized=bool(getattr(args, "base_image", ""))), + make_target("candidate", candidate_bin, work_dir, ports[8:], containerized=bool(getattr(args, "candidate_image", ""))), ] for target in targets: if target.work_dir.exists() and any(target.work_dir.iterdir()): @@ -407,6 +425,7 @@ def run_remote_case( work_dir, ports[:8], work_dir / "base" / "frontend-prom-store.toml", + containerized=bool(getattr(args, "base_image", "")), ), make_target( "candidate", @@ -414,6 +433,7 @@ def run_remote_case( work_dir, ports[8:], work_dir / "candidate" / "frontend-prom-store.toml", + containerized=bool(getattr(args, "candidate_image", "")), ), ] for target in targets: @@ -495,8 +515,8 @@ def run_otlp_case( raise ValueError("--otelgen-bin is required for otlp_trace_load") ports = allocate_ports(16) targets = [ - make_target("base", base_bin, work_dir, ports[:8]), - make_target("candidate", candidate_bin, work_dir, ports[8:]), + make_target("base", base_bin, work_dir, ports[:8], containerized=bool(getattr(args, "base_image", ""))), + make_target("candidate", candidate_bin, work_dir, ports[8:], containerized=bool(getattr(args, "candidate_image", ""))), ] for target in targets: if target.work_dir.exists() and any(target.work_dir.iterdir()): @@ -634,6 +654,106 @@ def write_failed_report(work_dir: Path, case_path: Path, error: Exception) -> No ) +def docker_target(image: str, root: Path, command: list[str]) -> int: + """Expose an image as a binary, including read-bench subcommands. + + The caller terminates a process group. Keep Docker clients in separate + sessions so cleanup can stop the container before SST inspection starts. + """ + name = f"query-regression-{uuid.uuid4().hex}" + + def interrupted(signum, _frame): + raise SystemExit(128 + signum) + + previous = {sig: signal.signal(sig, interrupted) for sig in (signal.SIGTERM, signal.SIGINT)} + try: + subprocess.run([ + "docker", "create", "--name", name, "--network", "host", + "--user", f"{os.getuid()}:{os.getgid()}", + "--volume", f"{root}:{root}", "--workdir", str(root), + "--entrypoint", "/greptime/bin/greptime", image, *command, + ], check=True, stdout=subprocess.DEVNULL, start_new_session=True) + attached = subprocess.run( + ["docker", "start", "--attach", name], check=False, start_new_session=True, + ) + if attached.returncode != 0: + return attached.returncode + # docker start's status is not the contract for the container's exit code. + result = subprocess.run( + ["docker", "inspect", "--format", "{{.State.ExitCode}}", name], + check=True, capture_output=True, text=True, start_new_session=True, + ) + return int(result.stdout.strip()) + finally: + for sig in previous: + signal.signal(sig, signal.SIG_IGN) + try: + subprocess.run( + ["docker", "stop", "--time", "5", name], check=False, + stdout=subprocess.DEVNULL, timeout=8, start_new_session=True, + ) + except subprocess.TimeoutExpired: + # Force-removal below still waits for the container to stop. + pass + finally: + try: + subprocess.run( + ["docker", "rm", "--force", "--volumes", name], check=True, + stdout=subprocess.DEVNULL, timeout=5, start_new_session=True, + ) + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) + + +def prepare_targets(args: argparse.Namespace) -> dict[str, Any]: + """Resolve images once and create binary-compatible launchers for all cases.""" + metadata = {} + images = {} + for name in ("base", "candidate"): + image = getattr(args, f"{name}_image") + ref = getattr(args, f"{name}_ref") + if not image: + metadata[name] = {"kind": "source", "ref": os.environ.get(f"VERIFIED_{name.upper()}_SHA") or ref} + continue + if sys.platform != "linux": + raise ValueError("Docker targets require Linux host networking and a local Docker daemon") + if image.startswith("-") or any(c.isspace() for c in image): + raise ValueError(f"invalid {name} image reference: {image!r}") + if image not in images: + result = subprocess.run( + ["docker", "image", "inspect", image], check=False, capture_output=True, text=True, + ) + if result.returncode != 0: + subprocess.run(["docker", "pull", image], check=True) + result = subprocess.run( + ["docker", "image", "inspect", image], check=True, capture_output=True, text=True, + ) + images[image] = json.loads(result.stdout)[0] + info = images[image] + image_id = info["Id"] + if not re.fullmatch(r"sha256:[0-9a-f]{64}", image_id): + raise ValueError(f"invalid Docker image ID: {image_id!r}") + args.work_dir = args.work_dir.resolve() + if ":" in str(args.work_dir): + raise ValueError("Docker target work directory cannot contain ':'") + launchers = args.work_dir / ".targets" + launchers.mkdir(parents=True, exist_ok=True) + launcher = launchers / name + command = [sys.executable, str(Path(__file__).resolve()), + "docker-target", image_id, str(args.work_dir)] + launcher.write_text("#!/bin/sh\nexec " + shlex.join(command) + ' "$@"\n', encoding="utf-8") + launcher.chmod(0o755) + setattr(args, f"{name}_bin", launcher) + setattr(args, f"{name}_ref", f"{image} ({image_id})") + metadata[name] = { + "kind": "docker", "image": image, "image_id": image_id, + "repo_digests": info.get("RepoDigests", []), + "network": "host", "user": f"{os.getuid()}:{os.getgid()}", + } + return metadata + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--cases", action="append", help="'all', 'heavy', or comma/space separated case paths") @@ -663,6 +783,8 @@ def main() -> int: parser.add_argument("--base-ref", default=os.environ.get("BASE_REF", "")) parser.add_argument("--candidate-ref", default=os.environ.get("CANDIDATE_REF", "")) parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT")) + parser.add_argument("--base-image", default=os.environ.get("BASE_IMAGE", "")) + parser.add_argument("--candidate-image", default=os.environ.get("CANDIDATE_IMAGE", "")) args = parser.parse_args() try: cases = split_cases(args.cases or [os.environ.get("CASE_PATHS", "all")]) @@ -671,6 +793,12 @@ def main() -> int: append_github_output(args.github_output, 1) return 0 + preparation_error = None + metadata = {} + try: + metadata = prepare_targets(args) + except Exception as err: + preparation_error = err reports: list[Path] = [] status = 0 for case in cases: @@ -678,11 +806,18 @@ def main() -> int: work_dir = args.work_dir / case_slug(case_path) reports.append(work_dir / "query-regression-report.json") try: + if preparation_error is not None: + raise preparation_error case_status = run_case(args, case_path, work_dir) except Exception as err: # noqa: BLE001 - preserve an aggregate report for the summary step print(f"error: query regression case {case_path}: {err}", flush=True) write_failed_report(work_dir, case_path, err) case_status = 1 + report_path = reports[-1] + if (args.base_image or args.candidate_image) and report_path.exists(): + report = json.loads(report_path.read_text(encoding="utf-8")) + report["execution_targets"] = metadata + report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") if case_status != 0: status = case_status or 1 @@ -695,4 +830,6 @@ def main() -> int: if __name__ == "__main__": + if len(sys.argv) > 1 and sys.argv[1] == "docker-target": + raise SystemExit(docker_target(sys.argv[2], Path(sys.argv[3]), sys.argv[4:])) raise SystemExit(main()) diff --git a/.github/scripts/update-query-regression-helpers-image.sh b/.github/scripts/update-query-regression-helpers-image.sh new file mode 100644 index 00000000000..9ef9122da74 --- /dev/null +++ b/.github/scripts/update-query-regression-helpers-image.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail +: "${HELPERS_IMAGE:?Verified image digest is required}" +: "${GITHUB_REPOSITORY:?Repository is required}" +repository=greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/query-regression-helpers +[[ "$HELPERS_IMAGE" == "$repository@sha256:"* ]] +[[ "${HELPERS_IMAGE#"$repository@"}" =~ ^sha256:[0-9a-f]{64}$ ]] +# Like QUERY_REGRESSION_ECS_IMAGE_ID, this is CI configuration, not a source +# change. Create on first publication, then update only after image verification. +gh variable set QUERY_REGRESSION_HELPERS_IMAGE \ + --repo "$GITHUB_REPOSITORY" --body "$HELPERS_IMAGE" +gh api "repos/$GITHUB_REPOSITORY/actions/variables/QUERY_REGRESSION_HELPERS_IMAGE" \ + | jq -e --arg expected "$HELPERS_IMAGE" '.value == $expected' >/dev/null +printf 'QUERY_REGRESSION_HELPERS_IMAGE updated to `%s`\n' "$HELPERS_IMAGE" diff --git a/.github/workflows/publish-query-regression-helpers.yml b/.github/workflows/publish-query-regression-helpers.yml new file mode 100644 index 00000000000..eca58cff96d --- /dev/null +++ b/.github/workflows/publish-query-regression-helpers.yml @@ -0,0 +1,166 @@ +name: Publish query regression helpers + +on: + workflow_dispatch: + workflow_call: + push: + branches: [main] + paths: + - src/cmd/src/bin/query_perf_fixture.rs + - src/cmd/src/bin/query_perf_fixture/** + - src/cmd/src/bin/query_regression_runner.rs + - src/cmd/src/bin/query_regression_runner/** + - docker/query-regression-helpers/** + - .github/workflows/publish-query-regression-helpers.yml + - .github/scripts/publish-query-regression-helpers.sh + - .github/scripts/build-query-regression-helpers.sh + - .github/scripts/update-query-regression-helpers-image.sh + - Cargo.lock + +permissions: + contents: read + +# Serialize publication and updates of the repository image variable. +concurrency: + group: publish-query-regression-helpers + cancel-in-progress: false + +jobs: + provision: + if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + label: ${{ steps.provision.outputs.label }} + instance_id: ${{ steps.provision.outputs.instance_id }} + runner_name: ${{ steps.provision.outputs.runner_name }} + steps: + - name: Checkout trusted provisioning scripts + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + persist-credentials: false + + - name: Provision ECS runner + id: provision + uses: ./.github/actions/aliyun-ecs-create + with: + access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} + access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} + vswitch-id: ${{ vars.ALIYUN_ECS_VSWITCH_ID }} + security-group-id: ${{ vars.ALIYUN_ECS_SECURITY_GROUP_ID }} + instance-type: ${{ vars.ALIYUN_ECS_INSTANCE_TYPE }} + # Preserve query regression's original resource and janitor budgets. + system-disk-gib: '80' + ttl-hours: '4' + enable-docker: 'true' + resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} + image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} + runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} + runner-gid: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} + + build: + needs: provision + runs-on: ${{ needs.provision.outputs.label }} + timeout-minutes: 150 + env: + CARGO_HOME: /home/runner/.cargo + CARGO_TARGET_DIR: /home/runner/query-regression-target + RUSTUP_HOME: /opt/rustup + RUSTUP_TOOLCHAIN: nightly-2026-03-21 + RUSTUP_AUTO_INSTALL: "0" + CARGO_INCREMENTAL: "0" + RUSTFLAGS: -D warnings -C link-arg=-fuse-ld=mold + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Build and validate helpers + run: bash .github/scripts/build-query-regression-helpers.sh + - name: Upload reusable helper image + uses: actions/upload-artifact@v4 + with: + name: helpers-image + path: helpers-image/ + compression-level: 1 + retention-days: 7 + if-no-files-found: error + - name: Upload build validation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: helpers-build-validation + path: helpers-validation/ + if-no-files-found: warn + + publish: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 45 + outputs: + image: ${{ steps.publish.outputs.image }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 + with: + name: helpers-image + path: helpers-image/ + - name: Login to registry + uses: docker/login-action@v3 + with: + registry: greptime-registry.cn-hangzhou.cr.aliyuncs.com + username: ${{ secrets.ALICLOUD_USERNAME }} + password: ${{ secrets.ALICLOUD_PASSWORD }} + - name: Publish and verify helpers + id: publish + run: bash .github/scripts/publish-query-regression-helpers.sh + - name: Upload publication validation evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: helpers-publish-validation + path: | + helpers-validation/digest.txt + helpers-validation/extracted/helpers-image.json + if-no-files-found: warn + + update-variable: + needs: publish + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Update helpers image repository variable + env: + GH_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + HELPERS_IMAGE: ${{ needs.publish.outputs.image }} + run: bash .github/scripts/update-query-regression-helpers-image.sh + + teardown: + if: ${{ always() && needs.provision.outputs.instance_id != '' }} + needs: [provision, build] + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout trusted teardown scripts + uses: actions/checkout@v4 + with: + repository: ${{ github.repository }} + ref: ${{ github.sha }} + persist-credentials: false + + - name: Teardown ECS runner + uses: ./.github/actions/aliyun-ecs-delete + with: + access-key-id: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_ID }} + access-key-secret: ${{ secrets.ALICLOUD_ECS_ACCESS_KEY_SECRET }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + region-id: ${{ vars.ALIYUN_ECS_REGION_ID }} + instance-id: ${{ needs.provision.outputs.instance_id }} + runner-name: ${{ needs.provision.outputs.runner_name }} diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index 6543281409d..d65d6666658 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -26,6 +26,21 @@ on: description: Candidate ref/sha to build; reusable release calls must provide an immutable full SHA required: true type: string + base_image: + description: Base Docker image tag/digest (empty builds base_ref from source) + required: false + type: string + default: "" + candidate_image: + description: Candidate Docker image tag/digest (empty builds candidate_ref from source) + required: false + type: string + default: "" + helpers_image: + description: Helpers image@sha256 or pinned from repository variable (empty builds from source) + required: false + type: string + default: "" cargo_profile: description: Cargo profile to build base and candidate binaries with required: false @@ -80,6 +95,21 @@ on: description: Runner HTTP timeout seconds required: true default: "300" + base_image: + description: Base Docker image tag/digest (empty builds base_ref from source) + required: false + type: string + default: "" + candidate_image: + description: Candidate Docker image tag/digest (empty builds candidate_ref from source) + required: false + type: string + default: "" + helpers_image: + description: Helpers image@sha256 or pinned from repository variable (empty builds from source) + required: false + type: string + default: "" cargo_profile: description: Cargo profile to build base and candidate binaries with required: true @@ -132,6 +162,7 @@ jobs: python3 tests/perf/test_query_regression_runner_otlp_trace_load.py python3 tests/perf/test_query_regression_summary_otlp.py python3 tests/perf/test_query_regression_case_selection.py + python3 tests/perf/test_query_regression_helpers.py python3 tests/perf/test_query_regression_nightly_refs.py python3 tests/perf/test_query_regression_slash.py python3 tests/perf/test_aliyun_ecs_runner_scripts.py @@ -171,6 +202,7 @@ jobs: # Preserve query regression's original resource and janitor budgets. system-disk-gib: '80' ttl-hours: '4' + enable-docker: ${{ inputs.base_image != '' || inputs.candidate_image != '' || inputs.helpers_image != '' }} resource-group-id: ${{ vars.ALIYUN_ECS_RESOURCE_GROUP_ID }} image-id: ${{ vars.QUERY_REGRESSION_ECS_IMAGE_ID }} runner-uid: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} @@ -214,6 +246,10 @@ jobs: PR_NUMBER: ${{ inputs.pr_number }} HEAD_REPO: ${{ inputs.head_repo }} BASE_REPO: ${{ inputs.base_repo }} + HELPERS_IMAGE: ${{ inputs.helpers_image }} + QUERY_REGRESSION_HELPERS_IMAGE: ${{ vars.QUERY_REGRESSION_HELPERS_IMAGE }} + BASE_IMAGE: ${{ inputs.base_image }} + CANDIDATE_IMAGE: ${{ inputs.candidate_image }} BASE_REF: ${{ inputs.base_ref }} CANDIDATE_REF: ${{ inputs.candidate_ref || github.ref }} EVENT_MERGE_SHA: ${{ inputs.pr_number != '' && inputs.candidate_ref || '' }} @@ -224,6 +260,16 @@ jobs: EXPECTED_RUNNER_UID: ${{ vars.QUERY_REGRESSION_RUNNER_UID || '1001' }} EXPECTED_RUNNER_GID: ${{ vars.QUERY_REGRESSION_RUNNER_GID || '1001' }} steps: + - name: Validate image target admission + run: | + if [[ "${PR_ADMISSION}" == "true" && ( -n "${BASE_IMAGE}" || -n "${CANDIDATE_IMAGE}" || -n "${HELPERS_IMAGE}" ) ]]; then + echo "PR admission requires source targets for verified commit attribution." >&2 + exit 1 + fi + if [[ -n "${BASE_IMAGE}" || -n "${CANDIDATE_IMAGE}" || -n "${HELPERS_IMAGE}" ]]; then + docker info >/dev/null + fi + - name: Report provisioned ECS runner if: ${{ needs.provision.outputs.instance_id != '' }} shell: bash @@ -418,6 +464,7 @@ jobs: > "${GITHUB_WORKSPACE}/query-regression-trusted-scripts/query-regression-summary.py" - name: Verify runner image tools + if: ${{ inputs.base_image == '' || inputs.candidate_image == '' || inputs.helpers_image == '' }} shell: bash run: | set -euo pipefail @@ -534,6 +581,7 @@ jobs: fi - name: Prepare query regression cache + if: ${{ inputs.base_image == '' || inputs.candidate_image == '' || inputs.helpers_image == '' }} shell: bash working-directory: src run: | @@ -694,6 +742,7 @@ jobs: sccache --show-stats - name: Build base greptime + if: ${{ inputs.base_image == '' }} working-directory: src run: | set -euo pipefail @@ -728,14 +777,27 @@ jobs: git reset --hard "${VERIFIED_CANDIDATE_SHA}" git clean -ffdx + - name: Extract precompiled query regression helpers + if: ${{ inputs.helpers_image != '' }} + working-directory: src + run: | + python3 .github/scripts/query-regression-helpers.py "$HELPERS_IMAGE" \ + "$GITHUB_WORKSPACE/query-regression-bins/candidate" + cat "$GITHUB_WORKSPACE/query-regression-bins/candidate/helpers-image.json" >> "$GITHUB_STEP_SUMMARY" + - name: Build candidate greptime and query regression helpers + if: ${{ inputs.candidate_image == '' || inputs.helpers_image == '' }} working-directory: src run: | set -euo pipefail SECONDS=0 - cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime - cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools - cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_regression_runner --features dev-tools + if [[ -z "${CANDIDATE_IMAGE}" ]]; then + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime + fi + if [[ -z "${HELPERS_IMAGE}" ]]; then + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_regression_runner --features dev-tools + fi candidate_build_elapsed="${SECONDS}" printf 'Candidate greptime and query regression helper cargo builds elapsed: %s seconds\n' "${candidate_build_elapsed}" printf -- '- Candidate greptime and query regression helper cargo builds: %s seconds\n' "${candidate_build_elapsed}" >> "${GITHUB_STEP_SUMMARY}" @@ -745,12 +807,16 @@ jobs: target_dir="debug" fi mkdir -p "${GITHUB_WORKSPACE}/query-regression-bins/candidate" - cp "${CARGO_TARGET_DIR}/${target_dir}/greptime" \ - "${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime" - cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \ - "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture" - cp "${CARGO_TARGET_DIR}/${target_dir}/query_regression_runner" \ - "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_regression_runner" + if [[ -z "${CANDIDATE_IMAGE}" ]]; then + cp "${CARGO_TARGET_DIR}/${target_dir}/greptime" \ + "${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime" + fi + if [[ -z "${HELPERS_IMAGE}" ]]; then + cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \ + "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture" + cp "${CARGO_TARGET_DIR}/${target_dir}/query_regression_runner" \ + "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_regression_runner" + fi - name: Run query regression id: run @@ -792,6 +858,7 @@ jobs: query-regression-work/**/logs/** query-regression-work/**/otelgen/** query-regression-summary.md + query-regression-bins/candidate/helpers-image.json machine-telemetry.log if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/release-dev-builder-images.yaml b/.github/workflows/release-dev-builder-images.yaml index 2e1041c1ff4..9f74035f8c9 100644 --- a/.github/workflows/release-dev-builder-images.yaml +++ b/.github/workflows/release-dev-builder-images.yaml @@ -35,6 +35,11 @@ on: description: Rebuild the query-regression ECS runner image (and update QUERY_REGRESSION_ECS_IMAGE_ID + RUNNER_IMAGE_EPOCH) required: false default: false + release_query_regression_helpers_image: + type: boolean + description: Publish precompiled query regression helpers and update the image repository variable + required: false + default: false update_dev_builder_image_tag: type: boolean description: Update the DEV_BUILDER_IMAGE_TAG in Makefile and create a PR @@ -451,3 +456,8 @@ jobs: echo "## Lockstep updates" >> "$GITHUB_STEP_SUMMARY" echo "- QUERY_REGRESSION_ECS_IMAGE_ID → \`${image_id}\`" >> "$GITHUB_STEP_SUMMARY" echo "- RUNNER_IMAGE_EPOCH → \`${new_epoch}\` (PR on \`${BRANCH}\`, pending review/merge)" >> "$GITHUB_STEP_SUMMARY" + + release-query-regression-helpers: + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && inputs.release_query_regression_helpers_image }} + uses: ./.github/workflows/publish-query-regression-helpers.yml + secrets: inherit diff --git a/docker/query-regression-helpers/Dockerfile b/docker/query-regression-helpers/Dockerfile new file mode 100644 index 00000000000..d1a93171695 --- /dev/null +++ b/docker/query-regression-helpers/Dockerfile @@ -0,0 +1,7 @@ +# Artifact image: binaries run on the query-regression ECS host after extraction. +# Build on the same ECS image as the consumer to preserve the runtime ABI. +FROM scratch +ARG SOURCE_REVISION +LABEL org.opencontainers.image.source="https://github.com/GreptimeTeam/greptimedb" +LABEL org.opencontainers.image.revision=$SOURCE_REVISION +COPY query_perf_fixture query_regression_runner / diff --git a/tests/perf/README.md b/tests/perf/README.md index 0361f598a3b..a7e698fe047 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -495,18 +495,68 @@ query_regression_runner measure --case --fixture-generator ## GitHub Actions `.github/workflows/query-regression.yml` provides an opt-in CI entrypoint for -query regression runs. It builds its own binaries for now: +query regression runs. By default, it builds its own binaries: - base `greptime` from the PR base commit, or `workflow_dispatch` `base_ref` - candidate `greptime`, `query_perf_fixture`, and `query_regression_runner` from the PR merge ref/current candidate checkout - outer driver and summary formatter from the candidate checkout -The workflow builds base and candidate `greptime` as normal release-equivalent -binaries. Candidate `query_perf_fixture` and `query_regression_runner` are the +For source targets, the workflow builds `greptime` as a normal release-equivalent +binary. Candidate `query_perf_fixture` and `query_regression_runner` are the extra head-side helpers; `finalize-remote` uses candidate `greptime datanode parquetbench/scanbench` as the read-bench tool against each target's data directory. +### Docker image targets + +`base_image` and `candidate_image` are independent optional inputs to manual +and reusable query-regression workflows. Leave an input empty to build that +side from source, or set it to a GreptimeDB image tag/digest to skip its server +build. All four source/image combinations are supported. PR slash-command runs +continue to use verified source revisions; image overrides are rejected for PR +admission so reports cannot attribute an image to a different commit. + +The candidate checkout still supplies cases and builds `query_perf_fixture` +and `query_regression_runner`, even with two image targets. `base_ref` and +`candidate_ref` remain required for reusable calls. Remote-write read-bench +commands use the candidate target, including its image when configured. + +The local driver accepts `--base-image` / `--candidate-image` (or `BASE_IMAGE` / +`CANDIDATE_IMAGE`). An image overrides that side's `--*-bin`. For example: + +```bash +python3 .github/scripts/query-regression-run.py \ + --candidate-src . \ + --base-image greptime/greptimedb:v1.3.0 \ + --candidate-bin /absolute/path/to/candidate/greptime \ + --fixture-generator /absolute/path/to/query_perf_fixture \ + --runner /absolute/path/to/query_regression_runner \ + --cases tests/perf/query_cases/smoke_direct_sst/case.toml \ + --summary-script .github/scripts/query-regression-summary.py +``` + +Image targets require Linux and a local Docker daemon accessible to the runner +user; the workflow checks access before building. For image targets, ECS +provisioning enables the existing `enable-docker` option: root cloud-init starts +the image-provided Docker daemon, grants the runner Docker group access, and +verifies access before starting the runner. Source-only runs keep this option +disabled. No ECS image rebuild is needed. Manually managed hosts must provision +Docker access and restart the runner service before selecting image targets. +Images must expose the GreptimeDB binary at `/greptime/bin/greptime` and support +the commands and data formats used by the selected cases. Images are pulled +when absent locally, then pinned to their inspected image ID for the entire +run; use a registry digest for repeatability across hosts. Reports include +requested images, resolved IDs and repository digests alongside source commit +identities. + +Containers use host networking and the runner's UID/GID. The run work directory +is bind-mounted at the same absolute path, preserving generated configuration, +SST materialization, logs and post-stop storage inspection. No additional CPU +or memory limits are imposed. Keep host resources and workloads comparable; +container execution and image build options can still affect timing. Use a +fresh work directory for each run. Container cleanup completes before the +lifecycle proceeds to storage inspection or a component restart. + The workflow runs when an allowlisted repository admin comments `/query-regression` on a non-draft PR. It does not rerun on pushes, ready-for-review, or reopen events. `/query-regression` runs the nine routine @@ -565,3 +615,46 @@ Additional SQL optimizer cases: `LIMIT`. - `sql_join_filter_order`: two direct-SST tables joined on a shared tag with time filters, aggregate ordering, and `LIMIT`. + +### Precompiled query-regression helpers + +`helpers_image` is optional in Query Regression (dispatch and reusable calls). +The empty default still builds both helpers from the candidate source. Set it to +`pinned` to use the `QUERY_REGRESSION_HELPERS_IMAGE` repository variable, or provide an explicit +`image@sha256:...` reference. With both database image inputs and a helper image, +the run skips all Cargo builds and build-cache preparation. PR comment admission +continues to require source builds. + +The helper image is an artifact image containing `query_perf_fixture` and +`query_regression_runner`. They are extracted and run on the host, so their child +processes can use the same database launchers and paths. Images are built on the +query-regression ECS runner; alternate runner hosts must have compatible Linux +x86_64 libraries. Extraction smoke-tests both executables and fails without a +source-build fallback. `helpers-image.json` in the report artifact records the +image digest, image ID, and source revision. Case files and orchestration still +come from the candidate ref; choose a compatible helper revision when changing +the case DSL. + +The **Publish query regression helpers** workflow builds and pushes to +`greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/query-regression-helpers`. +It runs manually or on main when helper sources, packaging, or Cargo.lock change. +After verifying the published digest, it creates or updates the +`QUERY_REGRESSION_HELPERS_IMAGE` Actions repository variable, using +`GH_PERSONAL_ACCESS_TOKEN` with Actions-variable write permission (the same +credential used to update the ECS runner image variable). No version-update PR +is needed. Subsequent runs with `helpers_image: pinned` use that digest; runs +with an explicit digest keep their chosen version, and empty inputs still build +from source. To roll back, set the variable to a previously verified digest. +No mutable `latest` tag is consumed. Local extraction with `pinned` reads the +same name from the environment. +The existing **Release builder images** workflow can also invoke publication via +`release_query_regression_helpers_image`, including for initial branch testing. + +Helper publication separates the ECS build from the hosted registry upload. +The ECS job smoke-tests the packaged binaries and compares fixture plans twice, +then retains the image archive and binary checksums as `helpers-image` for seven +days. The hosted job uses Skopeo (as in builder-image publishing), reads back the +published digest, and checks both binaries against those checksums. If upload +fails, rerun failed jobs to reuse the completed build artifact without compiling +again. Runtime compatibility is checked on ECS; the hosted publishing job only +compares bytes because its system libraries can differ. diff --git a/tests/perf/test_query_regression_helpers.py b/tests/perf/test_query_regression_helpers.py new file mode 100644 index 00000000000..329f34e0759 --- /dev/null +++ b/tests/perf/test_query_regression_helpers.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Checks for helper image extraction and unchanged source-build defaults.""" + +import hashlib +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location("helpers", ROOT / ".github/scripts/query-regression-helpers.py") +helpers = importlib.util.module_from_spec(spec) +spec.loader.exec_module(helpers) +IMAGE = "registry/tools/helpers@sha256:" + "a" * 64 +REVISION = "b" * 40 + + +class HelpersTest(unittest.TestCase): + def test_repository_variable_and_explicit_digest(self): + with patch.dict(os.environ, {"QUERY_REGRESSION_HELPERS_IMAGE": IMAGE}): + self.assertEqual(helpers.resolve_image("pinned"), IMAGE) + explicit = IMAGE.replace("a" * 64, "b" * 64) + self.assertEqual(helpers.resolve_image(explicit), explicit) + for value in ("", "registry/helpers:latest", IMAGE + "\nmalicious", "--help"): + with self.subTest(value=value), self.assertRaises(ValueError): + helpers.resolve_image(value) + for variables in ({}, {"QUERY_REGRESSION_HELPERS_IMAGE": "registry/helpers:latest"}): + with patch.dict(os.environ, variables, clear=True), self.assertRaises(ValueError): + helpers.resolve_image("pinned") + + def test_extraction_provenance_and_cleanup_on_failure(self): + for fail in (False, True): + with self.subTest(fail=fail), tempfile.TemporaryDirectory() as tmp: + destination = Path(tmp) + calls = [] + + def run(*args): + calls.append(args) + if args[:3] == ("docker", "image", "inspect"): + return json.dumps([{"Id": "sha256:image", "Config": {"Labels": { + "org.opencontainers.image.revision": REVISION}}}]) + if args[:2] == ("docker", "create"): + return "container" + if args[:2] == ("docker", "cp"): + if fail: + raise subprocess.CalledProcessError(1, args) + Path(args[-1]).write_bytes(b"binary") + return "" + + with patch.object(helpers, "run", side_effect=run): + if fail: + with self.assertRaises(subprocess.CalledProcessError): + helpers.extract(IMAGE, destination) + self.assertFalse((destination / "helpers-image.json").exists()) + else: + self.assertEqual(helpers.extract(IMAGE, destination), { + "image": IMAGE, "image_id": "sha256:image", "source_revision": REVISION}) + for binary in helpers.BINARIES: + self.assertIn((str(destination / binary), "--help"), calls) + self.assertEqual(calls[-1], ("docker", "rm", "--force", "--volumes", "container")) + + def test_missing_revision_fails_before_container_creation(self): + with tempfile.TemporaryDirectory() as tmp, patch.object( + helpers, "run", side_effect=["", '[{"Config": {}}]'] + ) as run: + with self.assertRaises(ValueError): + helpers.extract(IMAGE, Path(tmp)) + self.assertEqual(run.call_count, 2) + + def test_published_checksums_and_mismatch_cleanup(self): + for valid in (True, False): + with self.subTest(valid=valid), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + checksum = hashlib.sha256(b"binary" if valid else b"wrong").hexdigest() + checksums = root / "SHA256SUMS" + checksums.write_text("".join(f"{checksum} {name}\n" for name in helpers.BINARIES)) + calls = [] + + def run(*args): + calls.append(args) + if args[:3] == ("docker", "image", "inspect"): + return json.dumps([{"Id": "sha256:image", "Config": {"Labels": { + "org.opencontainers.image.revision": REVISION}}}]) + if args[:2] == ("docker", "create"): + return "container" + if args[:2] == ("docker", "cp"): + Path(args[-1]).write_bytes(b"binary") + return "" + + with patch.object(helpers, "run", side_effect=run): + if valid: + helpers.extract(IMAGE, root / "bins", checksums) + else: + with self.assertRaisesRegex(ValueError, "checksum mismatch"): + helpers.extract(IMAGE, root / "bins", checksums) + self.assertFalse((root / "bins/helpers-image.json").exists()) + self.assertTrue(all(call[0] == "docker" for call in calls)) + self.assertEqual(calls[-1], ("docker", "rm", "--force", "--volumes", "container")) + + def test_checksum_manifest_requires_both_helpers(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + checksums = root / "SHA256SUMS" + checksums.write_text("0" * 64 + " query_perf_fixture\n") + with patch.object(helpers, "run") as run, self.assertRaises(ValueError): + helpers.extract(IMAGE, root, checksums) + run.assert_not_called() + + def test_candidate_build_matrix(self): + # Execute the actual workflow shell with recorder commands. Changing + # only image inputs must preserve source defaults and skip opted-in builds. + import os + workflow = (ROOT / ".github/workflows/query-regression.yml").read_text() + step = workflow.split(" - name: Build candidate greptime and query regression helpers\n", 1)[1] + step = step.split(" - name:", 1)[0].split(" run: |\n", 1)[1] + script = "\n".join(line[10:] for line in step.splitlines()) + for candidate in ("", "server-image" ): + for image in ("", IMAGE): + with self.subTest(candidate=candidate, helpers=image), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + log = root / "commands" + for command in ("cargo", "cp", "sccache"): + path = root / command + path.write_text('#!/bin/sh\nprintf "%s\\n" "' + command + ' $*" >> "$COMMAND_LOG"\n') + path.chmod(0o755) + env = dict(os.environ, PATH=tmp + os.pathsep + os.environ["PATH"], + COMMAND_LOG=str(log), CANDIDATE_IMAGE=candidate, HELPERS_IMAGE=image, + CARGO_PROFILE="nightly", CARGO_TARGET_DIR=str(root / "target"), + GITHUB_WORKSPACE=tmp, GITHUB_STEP_SUMMARY=str(root / "summary")) + subprocess.run(["bash", "-c", script], env=env, check=True, capture_output=True) + builds = [line for line in log.read_text().splitlines() if line.startswith("cargo ")] + expected = ([] if candidate else ["greptime"]) + ([] if image else list(helpers.BINARIES)) + self.assertEqual([line.split("--bin ")[1].split()[0] for line in builds], expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/perf/test_query_regression_runner_compaction_toctou.py b/tests/perf/test_query_regression_runner_compaction_toctou.py index 58f7f31b7fa..120e74d22f4 100644 --- a/tests/perf/test_query_regression_runner_compaction_toctou.py +++ b/tests/perf/test_query_regression_runner_compaction_toctou.py @@ -13,15 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Regression coverage for the outer remote-write lifecycle.""" +"""Regression coverage for remote-write and Docker target lifecycles.""" import importlib.util import json import sys +import signal +import subprocess import tempfile import unittest from pathlib import Path -from unittest.mock import patch +from types import SimpleNamespace +from unittest.mock import Mock, patch RUNNER_PATH = Path(__file__).resolve().parents[2] / ".github/scripts/query-regression-run.py" @@ -110,5 +113,129 @@ class RemoteWriteLifecycleTest(unittest.TestCase): self.assertIn("stop:candidate:frontend", events) +class DockerTargetTest(unittest.TestCase): + image_id = "sha256:" + "a" * 64 + + def test_independent_target_selection_and_image_identity(self) -> None: + for base_image, candidate_image in (("", ""), ("image:v1", ""), ("", "image:v1"), ("image:v1", "image:v1")): + with self.subTest(base=base_image, candidate=candidate_image), tempfile.TemporaryDirectory() as tmp: + args = SimpleNamespace( + base_image=base_image, candidate_image=candidate_image, + base_bin=Path("base"), candidate_bin=Path("candidate"), + base_ref="base-sha", candidate_ref="candidate-sha", work_dir=Path(tmp), + ) + result = SimpleNamespace(returncode=0, stdout=json.dumps([ + {"Id": self.image_id, "RepoDigests": ["image@sha256:abc"]}, + ])) + with patch.object(runner.subprocess, "run", return_value=result) as run: + metadata = runner.prepare_targets(args) + self.assertEqual(run.call_count, int(bool(base_image or candidate_image))) + for name, image in (("base", base_image), ("candidate", candidate_image)): + if image: + launcher = getattr(args, f"{name}_bin") + self.assertTrue(launcher.stat().st_mode & 0o111) + self.assertIn(self.image_id, launcher.read_text()) + self.assertIn('"$@"', launcher.read_text()) + self.assertEqual(metadata[name]["image"], image) + self.assertEqual(metadata[name]["image_id"], self.image_id) + else: + self.assertEqual(getattr(args, f"{name}_bin"), Path(name)) + self.assertEqual(metadata[name], {"kind": "source", "ref": f"{name}-sha"}) + + def test_default_source_run_preserves_report_and_refs(self) -> None: + payload = '{"status":"ok","targets":[],"thresholds":[]}\n' + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + def run_case(args, _case_path, work_dir): + self.assertEqual((args.base_image, args.candidate_image), ("", "")) + self.assertEqual((args.base_ref, args.candidate_ref), ("main", "feature")) + self.assertEqual(args.cargo_profile, "nightly") + work_dir.mkdir(parents=True) + (work_dir / "query-regression-report.json").write_text(payload) + return 0 + argv = ["runner", "--cases", "case.toml", "--work-dir", str(root), + "--base-ref", "main", "--candidate-ref", "feature"] + with (patch.dict(runner.os.environ, {}, clear=True), + patch.object(sys, "argv", argv), + patch.object(runner, "run_case", side_effect=run_case), + patch.object(runner, "write_summary", return_value=0), + patch.object(runner.subprocess, "run") as subprocess_run): + self.assertEqual(runner.main(), 0) + subprocess_run.assert_not_called() + reports = list(root.rglob("query-regression-report.json")) + self.assertEqual(len(reports), 1) + self.assertEqual(reports[0].read_bytes(), payload.encode()) + + def test_missing_image_is_pulled_and_inspected(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + args = SimpleNamespace(base_image="image:v1", candidate_image="", base_ref="", candidate_ref="", work_dir=Path(tmp)) + results = [SimpleNamespace(returncode=1), SimpleNamespace(returncode=0), + SimpleNamespace(stdout=json.dumps([{"Id": self.image_id}]))] + with patch.object(runner.subprocess, "run", side_effect=results) as run: + runner.prepare_targets(args) + self.assertEqual([call.args[0][:2] for call in run.call_args_list], + [["docker", "image"], ["docker", "pull"], ["docker", "image"]]) + + def test_container_exit_code_and_cleanup(self) -> None: + commands = [] + def run(command, **kwargs): + commands.append(command) + self.assertTrue(kwargs["start_new_session"]) + return SimpleNamespace(returncode=0, stdout="42\n") + with patch.object(runner.subprocess, "run", side_effect=run): + status = runner.docker_target(self.image_id, Path("/tmp/work space"), ["datanode", "scanbench", "--help"]) + self.assertEqual(status, 42) + create = commands[0] + self.assertIn("/tmp/work space:/tmp/work space", create) + self.assertEqual(create[-4:], [self.image_id, "datanode", "scanbench", "--help"]) + self.assertEqual([cmd[1] for cmd in commands], ["create", "start", "inspect", "stop", "rm"]) + self.assertEqual(commands[-1][-1], create[3]) + + def test_signal_and_failed_start_both_remove_container(self) -> None: + for interruption in (True, False): + commands = [] + def run(command, **kwargs): + commands.append(command) + if command[1] == "start": + if interruption: + signal.getsignal(signal.SIGTERM)(signal.SIGTERM, None) + raise subprocess.CalledProcessError(1, command) + return SimpleNamespace(returncode=0) + previous = signal.getsignal(signal.SIGTERM) + with self.subTest(interruption=interruption), patch.object(runner.subprocess, "run", side_effect=run): + with self.assertRaises(SystemExit if interruption else subprocess.CalledProcessError): + runner.docker_target(self.image_id, Path("/tmp/work"), ["metasrv", "start"]) + self.assertEqual([cmd[1] for cmd in commands], ["create", "start", "stop", "rm"]) + self.assertEqual(signal.getsignal(signal.SIGTERM), previous) + + + def test_stop_timeout_still_force_removes(self) -> None: + commands = [] + def run(command, **kwargs): + commands.append(command) + if command[1] == "stop": + raise subprocess.TimeoutExpired(command, 8) + return SimpleNamespace(returncode=0, stdout="0") + with patch.object(runner.subprocess, "run", side_effect=run): + self.assertEqual(runner.docker_target(self.image_id, Path("/tmp/work"), ["--version"]), 0) + self.assertEqual(commands[-1][1], "rm") + + def test_failed_container_cleanup_is_not_ignored(self) -> None: + target = runner.make_target("base", Path("launcher"), Path("/tmp/work"), list(range(8)), containerized=True) + proc = Mock() + proc.poll.return_value = None + proc.returncode = 1 + with patch.object(runner.os, "killpg"): + with self.assertRaisesRegex(RuntimeError, "cleanup failed"): + runner.stop_component(target, "datanode", {("base", "datanode"): proc}) + + def test_cleanup_failure_does_not_skip_other_components(self) -> None: + target = runner.make_target("base", Path("launcher"), Path("/tmp/work"), list(range(8)), containerized=True) + with patch.object(runner, "stop_component", side_effect=[RuntimeError("failed"), None, None]) as stop: + with self.assertRaisesRegex(RuntimeError, "component cleanup failed"): + runner.stop_all([target], {}) + self.assertEqual([call.args[1] for call in stop.call_args_list], ["frontend", "datanode", "metasrv"]) + + if __name__ == "__main__": unittest.main()