From 31646a903fc88a038bdfdbdc69eed56eba2d56a9 Mon Sep 17 00:00:00 2001 From: WenyXu Date: Wed, 30 Sep 2026 02:08:35 +0000 Subject: [PATCH] ci: compare baseline and candidate GreptimeDB benchmarks Signed-off-by: WenyXu --- .../scripts/benchmark-version-comparison.py | 191 ++++++++++++++++++ .github/workflows/agent-observability.yml | 55 ++++- .github/workflows/tracesbench.yml | 59 +++++- .github/workflows/vmbench-long-range.yml | 27 ++- .../perf/test_benchmark_version_comparison.py | 124 ++++++++++++ tests/perf/test_ci_runner_scripts.py | 14 +- 6 files changed, 463 insertions(+), 7 deletions(-) create mode 100644 .github/scripts/benchmark-version-comparison.py create mode 100644 tests/perf/test_benchmark_version_comparison.py diff --git a/.github/scripts/benchmark-version-comparison.py b/.github/scripts/benchmark-version-comparison.py new file mode 100644 index 00000000000..1aa744df9ad --- /dev/null +++ b/.github/scripts/benchmark-version-comparison.py @@ -0,0 +1,191 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas.""" +import argparse +import html +import json +import math +import os +from pathlib import Path +import re +import statistics + + +def read(path): + return json.loads(path.read_text()) if path.is_file() else {} + + +def validate(env): + enabled = env.get('COMPARE_GREPTIMEDB', 'false') + if enabled not in ('true', 'false'): + raise ValueError('COMPARE_GREPTIMEDB must be true or false') + if enabled == 'false': + return + if env.get('RUN_GREPTIMEDB') != 'true': + raise ValueError('Version comparison requires Run GreptimeDB') + for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'): + if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')): + raise ValueError(f'Invalid {key}') + if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']: + raise ValueError('Baseline and candidate must have different image tags') + + +def same(a, b, fields): + return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields) + + +def finite(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def improvement(old, new, throughput=False): + if not finite(old) or not finite(new) or old <= 0: + return None + return (new / old - 1 if throughput else 1 - new / old) * 100 + + +def row(query, metric, old, new, compatible, reason): + delta = improvement(old, new, metric == 'QPS') if compatible else None + return dict(query=query, metric=metric, baseline=old, candidate=new, + improvement_percent=delta, status='comparable' if delta is not None else reason) + + +def agent(root): + def load(r): + return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json') + a, b = load(root / 'baseline'), load(root) + finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)] + compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished) + and a.get('passed') is True and b.get('passed') is True + and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs'))) + qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b)) + rows = [] + for q in sorted(qa.keys() | qb.keys()): + old, new = qa.get(q, {}), qb.get(q, {}) + ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0 + ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new)) + for key in ('p50_ms', 'p95_ms', 'p99_ms'): + rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result')) + return rows + + +def traces(root): + a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root)) + compatible = (a.get('success') is True and b.get('success') is True + and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus')) + and same(a.get('execution', {}), b.get('execution', {}), + ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight'))) + rows = [] + for q in (f'T{i:02}' for i in range(1, 9)): + values, identities, counts = [], [], [] + ok = compatible + for run in (a, b): + correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q] + all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q] + measured = [s for s in all_samples if s.get('phase') == 'measured'] + passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples) + expected = run.get('expected', {}).get('contracts', {}).get(q, {}) + passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None + and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples) + elapsed = [s.get('elapsed_ms') for s in measured] + passed = passed and all(finite(v) for v in elapsed) + ok = ok and passed + values.append(statistics.median(elapsed) if passed else None) + identities.append(expected) + counts.append((len(correctness), len(all_samples), len(measured))) + ok = ok and identities[0] == identities[1] and counts[0] == counts[1] + rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result')) + return rows + + +def long_range(root): + rows = [] + manifest = read(root / 'run-manifest.json') + healthy = all( + all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results' + else manifest.get(phase, {}).get(target, {})).get('state', '').split() + for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false')) + for phase in ('post_load_results', 'post_query_results') + for target in ('greptimedb-baseline', 'greptimedb')) + for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'): + a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json')) + for name in ('greptimedb-baseline', 'greptimedb')) + fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration', + 'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds') + fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds') + compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b)) + qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b)) + for q in sorted(qa.keys() | qb.keys()): + old, new = qa.get(q, {}), qb.get(q, {}) + ok = compatible and same(old, new, ('query', 'series', 'points')) + ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0 + and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new)) + for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',): + rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key, + old.get(key), new.get(key), ok, 'failed, missing or incompatible result')) + return rows + + +def render(document): + def cell(value): + return html.escape(str(value)).replace('|', '|').replace('\n', ' ') + def value(v, metric): + if not finite(v): + return '—' + if metric == 'QPS': + return f'{v:.3f}' + return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms' + lines = ['# GreptimeDB version comparison', '', + f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '', + 'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; ' + 'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. ' + 'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '', + '| Query | Metric | Baseline | Candidate | Improvement | Status |', + '| --- | --- | --- | --- | --- | --- |'] + for r in document['rows']: + delta = r['improvement_percent'] + lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']), + value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |') + if document.get('kind') == 'long-range': + lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.'] + if not document['rows']: + lines += ['', 'Results unavailable; comparison not performed.'] + return '\n'.join(lines) + '\n' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('validate', 'report')) + parser.add_argument('--kind', choices=('agent', 'traces', 'long-range')) + parser.add_argument('--root', type=Path) + args = parser.parse_args() + if args.command == 'validate': + validate(os.environ) + return + if args.root is None or args.kind is None: + parser.error('report requires --root and --kind') + document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''), + 'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''), + 'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)} + text = render(document) + # Preserve partial comparisons even if a target failed before creating output. + args.root.mkdir(parents=True, exist_ok=True) + (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n') + (args.root / 'version-comparison.md').write_text(text) + print(text, end='') + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index 0a96e65220c..a8e6d7510f4 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -26,8 +26,18 @@ on: - M default: S required: true + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -89,6 +99,8 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} PROFILE: ${{ inputs.dataset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }} @@ -118,6 +130,12 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" @@ -126,6 +144,12 @@ jobs: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py python3 greptimedb/tests/perf/test_ci_runner_scripts.py python3 greptimedb/tests/perf/test_agent_observability_summary.py + - name: Check runtime supports shared version-comparison data + if: inputs.compare_greptimedb + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" + docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \ + || { echo 'Comparison requires a runtime with --data-root support'; exit 1; } - name: Validate requested benchmark id: inputs shell: bash @@ -289,6 +313,9 @@ jobs: victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;; esac done + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE + fi jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \ --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \ --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \ @@ -312,6 +339,15 @@ jobs: --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json" + - name: Load and benchmark baseline GreptimeDB + id: baseline_greptimedb + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} + run: |- + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \ + --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" - name: Load and benchmark greptimedb id: greptimedb if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} @@ -357,6 +393,13 @@ jobs: GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ --owner "$OWNER" --root "$ARTIFACT_ROOT" fi + - name: Clean up baseline containers + if: always() && inputs.compare_greptimedb + run: |- + if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline" + fi - name: Benchmark summary if: always() run: | @@ -367,6 +410,16 @@ jobs: run: | python3 greptimedb/.github/scripts/agent-observability-summary.py \ --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: Baseline load / lifecycle summary + if: always() && inputs.compare_greptimedb + run: | + echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/agent-observability-summary.py \ + --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY" - name: Upload results (no dataset) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml index b44eb082642..e62373c4a3d 100644 --- a/.github/workflows/tracesbench.yml +++ b/.github/workflows/tracesbench.yml @@ -17,8 +17,18 @@ on: type: boolean default: false required: false + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -97,6 +107,8 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} DATASET: ${{ inputs.dataset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} RUNTIME_CPUS: ${{ inputs.runtime_cpus }} @@ -150,6 +162,12 @@ jobs: [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; } fi printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT" + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: | @@ -193,6 +211,10 @@ jobs: bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" metadata=$(docker image inspect "$RUNTIME_IMAGE") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \ + || { echo 'Comparison requires a runtime with --data-root support'; exit 1; } + fi revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata") [[ "$revision" =~ ^[0-9a-f]{40}$ ]] # Refuse runtimes without the trace runner and its validated dataset/report contract. @@ -323,12 +345,26 @@ jobs: printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" fi done + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image" + docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json" + resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json") + printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" + fi - name: Generate selected dataset once id: generate run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Baseline GreptimeDB load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ + --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ + --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: GreptimeDB load, correctness and benchmark if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} run: | @@ -361,12 +397,27 @@ jobs: python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Clean up baseline containers + if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ + --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Benchmark summary if: always() run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" - name: Load / lifecycle summary if: always() run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: Baseline load / lifecycle summary + if: always() && inputs.compare_greptimedb + run: | + echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/tracesbench-summary.py \ + --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | @@ -374,11 +425,15 @@ jobs: if [[ -d "$METADATA_ROOT" ]]; then rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/" fi - for entry in runs report report.json report.md; do + for entry in runs report report.json report.md version-comparison.json version-comparison.md; do if [[ -e "$RUN_ROOT/$entry" ]]; then rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/" fi done + if [[ -d "$RUN_ROOT/baseline/runs" ]]; then + mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline" + rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/" + fi - name: Upload HTML and results (no corpus) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/vmbench-long-range.yml b/.github/workflows/vmbench-long-range.yml index 60f3355542c..81e8de020dc 100644 --- a/.github/workflows/vmbench-long-range.yml +++ b/.github/workflows/vmbench-long-range.yml @@ -20,8 +20,18 @@ on: - 1mo default: 1w required: true + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -79,6 +89,8 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} RANGE_PRESET: ${{ inputs.range_preset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} @@ -107,6 +119,12 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" @@ -146,6 +164,9 @@ jobs: args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --run-classification controlled_server_benchmark --no-open-browser) + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range") + fi # Tags become target-spec fields; reject separators before passing them to the planner. for target in greptimedb victoriametrics; do case "$target" in @@ -303,6 +324,10 @@ jobs: if: always() run: | docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | diff --git a/tests/perf/test_benchmark_version_comparison.py b/tests/perf/test_benchmark_version_comparison.py new file mode 100644 index 00000000000..3af2e19ca8f --- /dev/null +++ b/tests/perf/test_benchmark_version_comparison.py @@ -0,0 +1,124 @@ +import copy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parents[2] / '.github/scripts' +spec = importlib.util.spec_from_file_location('comparison', SCRIPTS / 'benchmark-version-comparison.py') +comparison = importlib.util.module_from_spec(spec) +spec.loader.exec_module(comparison) + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + +class VersionComparison(unittest.TestCase): + def test_opt_in_validation(self): + comparison.validate({}) + comparison.validate({'COMPARE_GREPTIMEDB': 'false', 'GREPTIMEDB_TAG': 'invalid\n'}) + good = dict(COMPARE_GREPTIMEDB='true', RUN_GREPTIMEDB='true', + BASELINE_GREPTIMEDB_TAG='v1.2.1', GREPTIMEDB_TAG='v1.3.0-beta.1') + comparison.validate(good) + for key, value in [('RUN_GREPTIMEDB', 'false'), ('BASELINE_GREPTIMEDB_TAG', ''), + ('BASELINE_GREPTIMEDB_TAG', 'v1.3.0-beta.1'), + ('BASELINE_GREPTIMEDB_TAG', 'a,image=other'), ('GREPTIMEDB_TAG', 'x\ny'), + ('COMPARE_GREPTIMEDB', 'yes')]: + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + comparison.validate(good | {key: value}) + + def test_improvement_direction(self): + self.assertAlmostEqual(comparison.improvement(100, 80), 20) + self.assertAlmostEqual(comparison.improvement(100, 120), -20) + self.assertAlmostEqual(comparison.improvement(100, 120, True), 20) + for old, new in [(0, 3), (None, 4), (1, -1), (1, float('nan')), (True, 2)]: + self.assertIsNone(comparison.improvement(old, new)) + + def test_agent_result_gates(self): + summary = dict(passed=True, contract_sha256='contract', data_profile='S', query_window=[1, 2], + warmup_runs=1, measured_runs=5, per_query=[dict(query_id='Q01', result_fingerprint='answer', + success=5, errors=0, timeouts=0, p50_ms=100, p95_ms=100, p99_ms=100)]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + base = root / 'baseline/greptimedb-1/greptimedb-1/measured-summary.json' + candidate = root / 'greptimedb-1/greptimedb-1/measured-summary.json' + for r in (root / 'baseline', root): + (r / 'greptimedb-1').mkdir(parents=True, exist_ok=True) + (r / 'greptimedb-1/exit-code.txt').write_text('0\n') + write(base, summary) + changed = copy.deepcopy(summary); changed['per_query'][0]['p50_ms'] = 80 + write(candidate, changed) + self.assertAlmostEqual(comparison.agent(root)[0]['improvement_percent'], 20) + for key, value in [('errors', 1), ('timeouts', 1), ('result_fingerprint', 'wrong'), ('success', 0)]: + broken = copy.deepcopy(changed); broken['per_query'][0][key] = value + write(candidate, broken) + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + (root / 'greptimedb-1/exit-code.txt').write_text('1\n') + write(candidate, changed) + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + (root / 'greptimedb-1/exit-code.txt').write_text('0\n') + changed['query_window'] = [2, 3]; write(candidate, changed) + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + candidate.unlink() + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + + def test_traces_uses_measured_only_and_rejects_failed_query(self): + answer = dict(query_id='T01', outcome='passed', fingerprint='answer', row_count=1, elapsed_ms=100) + sample = dict(answer, phase='measured') + record = dict(success=True, expected={'contracts': {'T01': {'fingerprint': 'answer', 'row_count': 1}}}, + protocol={'runs': 5}, audit_expected={}, corpus='sanity', + execution=dict(db_cpus='8', db_memory='16g', load_workers_requested=4, max_in_flight=8), + correctness=[answer], samples=[dict(sample, phase='warmup', elapsed_ms=99999), sample, sample]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp);base = root / 'baseline/runs/greptimedb/run.json'; candidate = root / 'runs/greptimedb/run.json' + write(base, record); changed = copy.deepcopy(record) + for s in changed['samples']: + if s['phase'] == 'measured': s['elapsed_ms'] = 80 + write(candidate, changed) + self.assertAlmostEqual(comparison.traces(root)[0]['improvement_percent'], 20) + self.assertIsNone(comparison.traces(root)[1]['improvement_percent']) + for key, value in [('outcome', 'failed'), ('fingerprint', 'wrong'), ('row_count', 2)]: + broken = copy.deepcopy(changed); broken['samples'][1][key] = value; write(candidate, broken) + self.assertIsNone(comparison.traces(root)[0]['improvement_percent']) + + def test_long_range_gates(self): + summary = dict(workload='vmbench-long-range', query_file_sha256='hash', start='start', end='end', step='30m', + profile='small', data_duration='32d', scrape_interval='30s', cache_policy_effective='warm', + mode='serial', connection_mode='force-close', request_timeout_seconds=120, tries=5, warmup_runs=1, + errors=0, timeouts=0, per_query=[dict(query_index=1, query='up', series=1, points=2, + success=5, errors=0, timeouts=0, p50_ms=100)]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); base = root/'greptimedb-baseline/results-vmbench-long-range/serial-summary.json' + candidate = root/'greptimedb/results-vmbench-long-range/serial-summary.json' + health = {phase: {target: {'state': 'Running=true ExitCode=0 OOMKilled=false'} + for target in ('greptimedb-baseline', 'greptimedb')} + for phase in ('post_load_results', 'post_query_results')} + health['post_load_results'] = {k: {'docker': v} for k, v in health['post_load_results'].items()} + write(root / 'run-manifest.json', health) + write(base, summary); write(candidate, summary) + self.assertEqual(comparison.long_range(root)[0]['improvement_percent'], 0) + for key, value in [('start','different'),('tries',1),('query_file_sha256','other'),('errors',1)]: + write(candidate, summary | {key:value}) + self.assertIsNone(comparison.long_range(root)[0]['improvement_percent']) + write(candidate, summary) + broken=copy.deepcopy(health);broken['post_query_results']['greptimedb']['state']='Running=false ExitCode=137 OOMKilled=true' + write(root / 'run-manifest.json', broken) + self.assertIsNone(comparison.long_range(root)[0]['improvement_percent']) + write(root / 'run-manifest.json', health) + empty=copy.deepcopy(summary);empty['per_query'][0]['points']=0;write(candidate,empty) + self.assertIsNone(comparison.long_range(root)[0]['improvement_percent']) + + def test_missing_results_and_rendering(self): + with tempfile.TemporaryDirectory() as tmp: + for load in (comparison.agent, comparison.traces, comparison.long_range): + self.assertTrue(all(r['improvement_percent'] is None for r in load(Path(tmp)))) + rendered = comparison.render(dict(baseline_tag='old', candidate_tag='', rows=[])) + self.assertIn('<new|tag>', rendered) + self.assertIn('Results unavailable', rendered) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/perf/test_ci_runner_scripts.py b/tests/perf/test_ci_runner_scripts.py index 546ee06978c..c703e41c62b 100644 --- a/tests/perf/test_ci_runner_scripts.py +++ b/tests/perf/test_ci_runner_scripts.py @@ -553,10 +553,13 @@ class BenchmarkImagePullTest(unittest.TestCase): ''') images[0] = 'greptime/greptimedb:v1.3.0-beta.1' self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1', - GITHUB_ENV=str(self.root / 'env')) - for selected in (('greptimedb',), ('tempo',), ('greptimedb', 'victoriatraces', 'tempo')): + GITHUB_ENV=str(self.root / 'env'), BASELINE_GREPTIMEDB_TAG='v1.2.1') + for selected, compare in ((('greptimedb',), False), (('tempo',), False), + (('greptimedb', 'victoriatraces', 'tempo'), False), + (('greptimedb',), True), (('greptimedb', 'victoriatraces', 'tempo'), True)): with self.subTest(selected=selected): self.env['TARGETS'] = ','.join(selected) + self.env['COMPARE_GREPTIMEDB'] = str(compare).lower() (self.root / 'env').write_text('') (self.root / 'docker.log').unlink(missing_ok=True) result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env, @@ -564,10 +567,15 @@ class BenchmarkImagePullTest(unittest.TestCase): self.assertEqual(result.returncode, 0, result.stderr) selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images) if target in selected] + if compare: + selected_images.append('greptime/greptimedb:v1.2.1') self.assertEqual(self.lines('docker'), [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')]) expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n' - self.assertEqual((self.root / 'env').read_text(), expected_env if 'greptimedb' in selected else '') + expected_env = expected_env if 'greptimedb' in selected else '' + if compare: + expected_env += 'BASELINE_GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n' + self.assertEqual((self.root / 'env').read_text(), expected_env) (self.root / 'docker.log').unlink() self.env['PULL_FAIL_IMAGE'] = images[1] result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,