From 6e1a1374111887fa31df1b9573e27d37a2b06329 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Fri, 9 Oct 2026 12:39:51 +0000 Subject: [PATCH] ci: enhance benchmark workflows with version comparisons and diagnostics (#9383) * ci: update Agent and Traces benchmark runtime Signed-off-by: WenyXu * ci: use unrestricted benchmark runtimes by default Signed-off-by: WenyXu * ci: retry benchmark image pulls before generating datasets Signed-off-by: WenyXu * ci: increase tracesbench defaults to 8 CPU and 32 GiB Signed-off-by: WenyXu * ci: configure tracesbench query timeout and update runtime Signed-off-by: WenyXu * ci: update tracesbench runtime and configure import scan workers Signed-off-by: WenyXu * ci: use shared parallel tracesbench import preparation Signed-off-by: WenyXu * ci: update tracesbench runtime to preserve Tempo trace requests Signed-off-by: WenyXu * ci: summarize tracesbench query results and lifecycle Signed-off-by: WenyXu * ci: show benchmark resource usage and update parallel loader runtime Signed-off-by: WenyXu * fix(ci): resolve shared summary module in existing test harness Signed-off-by: WenyXu * ci: select tracesbench engines and GreptimeDB image version Signed-off-by: WenyXu * ci: update benchmark runtime with parallel Agent preparation Signed-off-by: WenyXu * ci: select benchmark runtime registry by cloud provider Signed-off-by: WenyXu * ci: retry long-range benchmark image pulls Signed-off-by: WenyXu * ci: compare baseline and candidate GreptimeDB benchmarks Signed-off-by: WenyXu * ci: pin published dual-version benchmark runtime Signed-off-by: WenyXu * ci: prepare agent benchmark indexes with updated runtime Signed-off-by: WenyXu * ci: validate agent admission and render traces version comparison Signed-off-by: WenyXu * ci: use runtime with traces index preparation Signed-off-by: WenyXu * ci: use consolidated runtime for benchmark version reports Signed-off-by: WenyXu * test: add license header to benchmark comparison tests Signed-off-by: WenyXu * fix(ci): read legacy Tempo ingestion metrics in traces summaries Signed-off-by: WenyXu * ci: require tracesbench query plan artifacts Signed-off-by: WenyXu * ci: use optimized tracesbench report runtime Signed-off-by: WenyXu * ci: update tracesbench runtime to merged report validation Signed-off-by: WenyXu --------- Signed-off-by: WenyXu --- .../scripts/agent-observability-summary.py | 3 + .../scripts/benchmark-version-comparison.py | 191 +++++++++++++++ .github/scripts/benchmark_resource_summary.py | 99 ++++++++ .github/scripts/pull-benchmark-image.sh | 25 ++ .github/scripts/tracesbench-summary.py | 169 +++++++++++++ .github/workflows/agent-observability.yml | 69 +++++- .github/workflows/checks.yml | 1 + .github/workflows/tracesbench.yml | 226 +++++++++++++++--- .github/workflows/vmbench-long-range.yml | 41 +++- .../perf/test_agent_observability_summary.py | 2 + .../perf/test_benchmark_version_comparison.py | 138 +++++++++++ tests/perf/test_ci_runner_scripts.py | 210 ++++++++++++++++ tests/perf/test_tracesbench_summary.py | 54 +++++ 13 files changed, 1187 insertions(+), 41 deletions(-) create mode 100644 .github/scripts/benchmark-version-comparison.py create mode 100644 .github/scripts/benchmark_resource_summary.py create mode 100644 .github/scripts/pull-benchmark-image.sh create mode 100644 .github/scripts/tracesbench-summary.py create mode 100644 tests/perf/test_benchmark_version_comparison.py create mode 100644 tests/perf/test_tracesbench_summary.py diff --git a/.github/scripts/agent-observability-summary.py b/.github/scripts/agent-observability-summary.py index 38ba4e9dbdd..33ba3ad7e6e 100644 --- a/.github/scripts/agent-observability-summary.py +++ b/.github/scripts/agent-observability-summary.py @@ -18,6 +18,8 @@ import json import math from pathlib import Path +from benchmark_resource_summary import render_resources + TARGETS = ('greptimedb', 'clickhouse', 'victorialogs') @@ -201,6 +203,7 @@ def lifecycle(root, targets): rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else 'invalid timestamp units (legacy artifact)', entry.get('exit_code')]) lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), ''] + lines.append(render_resources([('dataset', root)] + [(t, root / f'{t}-1') for t in targets], table)) return '\n'.join(lines) diff --git a/.github/scripts/benchmark-version-comparison.py b/.github/scripts/benchmark-version-comparison.py new file mode 100644 index 00000000000..1aa744df9ad --- /dev/null +++ b/.github/scripts/benchmark-version-comparison.py @@ -0,0 +1,191 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas.""" +import argparse +import html +import json +import math +import os +from pathlib import Path +import re +import statistics + + +def read(path): + return json.loads(path.read_text()) if path.is_file() else {} + + +def validate(env): + enabled = env.get('COMPARE_GREPTIMEDB', 'false') + if enabled not in ('true', 'false'): + raise ValueError('COMPARE_GREPTIMEDB must be true or false') + if enabled == 'false': + return + if env.get('RUN_GREPTIMEDB') != 'true': + raise ValueError('Version comparison requires Run GreptimeDB') + for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'): + if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')): + raise ValueError(f'Invalid {key}') + if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']: + raise ValueError('Baseline and candidate must have different image tags') + + +def same(a, b, fields): + return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields) + + +def finite(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def improvement(old, new, throughput=False): + if not finite(old) or not finite(new) or old <= 0: + return None + return (new / old - 1 if throughput else 1 - new / old) * 100 + + +def row(query, metric, old, new, compatible, reason): + delta = improvement(old, new, metric == 'QPS') if compatible else None + return dict(query=query, metric=metric, baseline=old, candidate=new, + improvement_percent=delta, status='comparable' if delta is not None else reason) + + +def agent(root): + def load(r): + return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json') + a, b = load(root / 'baseline'), load(root) + finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)] + compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished) + and a.get('passed') is True and b.get('passed') is True + and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs'))) + qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b)) + rows = [] + for q in sorted(qa.keys() | qb.keys()): + old, new = qa.get(q, {}), qb.get(q, {}) + ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0 + ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new)) + for key in ('p50_ms', 'p95_ms', 'p99_ms'): + rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result')) + return rows + + +def traces(root): + a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root)) + compatible = (a.get('success') is True and b.get('success') is True + and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus')) + and same(a.get('execution', {}), b.get('execution', {}), + ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight'))) + rows = [] + for q in (f'T{i:02}' for i in range(1, 9)): + values, identities, counts = [], [], [] + ok = compatible + for run in (a, b): + correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q] + all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q] + measured = [s for s in all_samples if s.get('phase') == 'measured'] + passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples) + expected = run.get('expected', {}).get('contracts', {}).get(q, {}) + passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None + and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples) + elapsed = [s.get('elapsed_ms') for s in measured] + passed = passed and all(finite(v) for v in elapsed) + ok = ok and passed + values.append(statistics.median(elapsed) if passed else None) + identities.append(expected) + counts.append((len(correctness), len(all_samples), len(measured))) + ok = ok and identities[0] == identities[1] and counts[0] == counts[1] + rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result')) + return rows + + +def long_range(root): + rows = [] + manifest = read(root / 'run-manifest.json') + healthy = all( + all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results' + else manifest.get(phase, {}).get(target, {})).get('state', '').split() + for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false')) + for phase in ('post_load_results', 'post_query_results') + for target in ('greptimedb-baseline', 'greptimedb')) + for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'): + a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json')) + for name in ('greptimedb-baseline', 'greptimedb')) + fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration', + 'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds') + fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds') + compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b)) + qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b)) + for q in sorted(qa.keys() | qb.keys()): + old, new = qa.get(q, {}), qb.get(q, {}) + ok = compatible and same(old, new, ('query', 'series', 'points')) + ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0 + and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new)) + for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',): + rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key, + old.get(key), new.get(key), ok, 'failed, missing or incompatible result')) + return rows + + +def render(document): + def cell(value): + return html.escape(str(value)).replace('|', '|').replace('\n', ' ') + def value(v, metric): + if not finite(v): + return '—' + if metric == 'QPS': + return f'{v:.3f}' + return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms' + lines = ['# GreptimeDB version comparison', '', + f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '', + 'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; ' + 'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. ' + 'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '', + '| Query | Metric | Baseline | Candidate | Improvement | Status |', + '| --- | --- | --- | --- | --- | --- |'] + for r in document['rows']: + delta = r['improvement_percent'] + lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']), + value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |') + if document.get('kind') == 'long-range': + lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.'] + if not document['rows']: + lines += ['', 'Results unavailable; comparison not performed.'] + return '\n'.join(lines) + '\n' + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('validate', 'report')) + parser.add_argument('--kind', choices=('agent', 'traces', 'long-range')) + parser.add_argument('--root', type=Path) + args = parser.parse_args() + if args.command == 'validate': + validate(os.environ) + return + if args.root is None or args.kind is None: + parser.error('report requires --root and --kind') + document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''), + 'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''), + 'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)} + text = render(document) + # Preserve partial comparisons even if a target failed before creating output. + args.root.mkdir(parents=True, exist_ok=True) + (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n') + (args.root / 'version-comparison.md').write_text(text) + print(text, end='') + + +if __name__ == '__main__': + main() diff --git a/.github/scripts/benchmark_resource_summary.py b/.github/scripts/benchmark_resource_summary.py new file mode 100644 index 00000000000..3a0a896c46e --- /dev/null +++ b/.github/scripts/benchmark_resource_summary.py @@ -0,0 +1,99 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render shared O11yBench sampled resource artifacts for lifecycle summaries.""" +import json +import math + + +def number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def metric(values, key, statistic): + entry = values.get(key, {}) + value = entry.get(statistic) + return value if number(entry.get('samples')) and entry['samples'] > 0 and number(value) else None + + +def decimal(value, suffix=''): + return f'{value:.2f}{suffix}' if number(value) else '—' + + +def memory(value): + if not number(value): + return '—' + for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'): + if value < 1024 or unit == 'TiB': + return decimal(value, f' {unit}') + value /= 1024 + + +def capacities(path): + # Quotas are metadata, not recomputed CPU statistics. Bound reads on long runs. + result = {} + try: + with path.open() as source: + for index, line in enumerate(source): + if index == 256: + break + record = json.loads(line) + cores = record.get('host', {}).get('cpu_count') + if number(cores) and cores > 0: + result.setdefault('host', cores) + for role, values in record.get('containers', {}).items(): + quota = values.get('configured_cpu_limit_cores') + if number(quota): + result.setdefault(role, quota) + if all(role in result for role in ('host', 'runtime', 'db')): + break + except (OSError, ValueError, TypeError, AttributeError): + pass + return result + + +def render_resources(scopes, table): + rows = [] + for scope, directory in scopes: + try: + summary = json.loads((directory / 'resources.summary.json').read_text()) + phases = summary.get('phases', {}) if summary.get('schema') == 'o11ybench-resources-v1' else {} + except (OSError, ValueError, TypeError, AttributeError): + phases = {} + if not isinstance(phases, dict) or not phases: + rows.append([scope, 'unavailable', *['—'] * 9]) + continue + limits = capacities(directory / 'resources.jsonl') + for phase, data in phases.items(): + for role in ('host', 'runtime', 'db'): + values = data.get('host', {}) if role == 'host' else data.get('containers', {}).get(role, {}) + if role == 'db' and scope == 'dataset': + continue + quota = limits.get(role) + cpu_samples = values.get('cpu_used_cores', {}).get('samples') + rows.append([scope, phase, role, cpu_samples, + decimal(metric(values, 'cpu_used_cores', 'mean')), + decimal(metric(values, 'cpu_used_cores', 'max')), + 'unlimited' if quota == 0 else decimal(quota), + memory(metric(values, 'memory_used_bytes' if role == 'host' else 'memory_usage_bytes', 'max')), + decimal(metric(values, 'io_wait_percent', 'mean'), '%'), + decimal(metric(values, 'io_wait_percent', 'max'), '%'), + memory(metric(values, 'swap_used_bytes', 'max'))]) + return '\n'.join(['## Sampled resources', '', + 'CPU is measured in cores. Mean/max summarize available samples, not continuous peaks; ' + 'short phases may have no CPU sample. CPU capacity is host cores or the configured container quota. ' + 'Host memory is total minus available; Docker memory excludes cache where supported. ' + 'Swap is host-wide usage, not swap activity. Missing evidence stays —.', '', + table(['Scope', 'Phase', 'Component', 'CPU samples', 'Mean CPU cores', 'Max CPU cores', + 'CPU capacity', 'Max memory', 'Mean I/O wait', 'Max I/O wait', 'Max swap'], rows), '']) diff --git a/.github/scripts/pull-benchmark-image.sh b/.github/scripts/pull-benchmark-image.sh new file mode 100644 index 00000000000..1a26a4aeaee --- /dev/null +++ b/.github/scripts/pull-benchmark-image.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Pull only: never retry database writes, benchmarks, or cloud provisioning. +set -euo pipefail + +if [[ $# != 1 || -z "$1" || "$1" == -* ]]; then + echo "Usage: $0 IMAGE" >&2 + exit 2 +fi +image=$1 +attempts=3 +for ((attempt=1; attempt<=attempts; attempt++)); do + echo "Pulling $image (attempt $attempt/$attempts)" >&2 + if timeout --kill-after=10s 180s docker pull "$image"; then + exit 0 + else + status=$? + fi + if ((attempt == attempts)); then + echo "Failed to pull $image after $attempts attempts (exit $status)" >&2 + exit "$status" + fi + delay=$((attempt * 10)) + echo "Pull failed (exit $status); retrying in ${delay}s" >&2 + sleep "$delay" +done diff --git a/.github/scripts/tracesbench-summary.py b/.github/scripts/tracesbench-summary.py new file mode 100644 index 00000000000..087a67498f4 --- /dev/null +++ b/.github/scripts/tracesbench-summary.py @@ -0,0 +1,169 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import json +import html +import math +from pathlib import Path + +from benchmark_resource_summary import render_resources + +TARGETS = ('greptimedb', 'victoriatraces', 'tempo') + + +def duration(ms): + if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0: + return '—' + if ms < 1: + return f'{ms * 1000:.0f} µs' + if ms < 1000: + return f'{ms:.1f} ms' + if ms < 60_000: + return f'{ms / 1000:.2f} s' + minutes, seconds = divmod(round(ms / 1000), 60) + hours, minutes = divmod(minutes, 60) + return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s' + + +def cell(value): + return html.escape(str(value if value is not None else '—')).replace('|', '|').replace('\n', ' ') + + +def read(path): + if not path.exists(): + return {} + return json.loads(path.read_text()) + + +def table(headers, rows): + return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |', + '| ' + ' | '.join('---' for _ in headers) + ' |'] + + ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows]) + + +def delta(value, baseline): + if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)): + return '—' + return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—' + + + +def load(root): + report = read(root / 'report.json') + runs = [read(p) for p in sorted((root / 'runs').glob('*/run.json'))] + return report, runs + + +def benchmark(report, runs, targets=TARGETS): + lines = ['# Tracesbench benchmark summary', '', + 'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. ' + 'Min/median/max come from the validated report, not recalculated samples. ' + 'Counts exclude warmup. Missing or excluded results have no latency comparison.', + '', f'Classification: {cell(report.get("classification"))}. ' + 'See the uploaded HTML for query semantics and client-side compensation.', ''] + statistics = {} + for corpus in report.get('per_corpus', []): + for stat in corpus.get('statistics', []): + if stat['target'] not in targets: + continue + statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat + rows, details = [], [] + for target in targets: + selected = [run for run in runs if run.get('target') == target] + if not selected: + lines.append(f'- **{target}**: unavailable / not run.') + for run in selected: + lines.append(f'- **{target}**: lifecycle ' + f'{"completed" if run.get("success") is True else "failed / incomplete"}; ' + f'query failures: {len(run.get("query_failures", []))}.') + for qid in (f'T{i:02}' for i in range(1, 9)): + correct = [q for q in run.get('correctness', []) if q.get('query_id') == qid] + measured = [q for q in run.get('samples', []) + if q.get('query_id') == qid and q.get('phase') == 'measured'] + errors = [q for q in correct + run.get('samples', []) + if q.get('query_id') == qid and q.get('outcome') not in ('passed', 'not_executed')] + reason = (run.get('exclusions', {}).get(qid) or + run.get('execution', {}).get('query_skips', {}).get(qid)) + stat = statistics.get((run.get('corpus'), run.get('round'), target, qid), {}) + base = statistics.get((run.get('corpus'), run.get('round'), 'greptimedb', qid), {}) + status = ('failed' if errors else 'excluded' if reason else + 'passed' if stat else + 'correctness passed; report unavailable' if correct and + all(q.get('outcome') == 'passed' for q in correct) else 'unavailable / incomplete') + rows.append([run.get('corpus'), run.get('round'), qid, target, status, + duration(stat.get('min_ms')), duration(stat.get('median_ms')), + duration(stat.get('max_ms')), + delta(stat.get('median_ms'), base.get('median_ms')) if target != 'greptimedb' else '—', + sum(q.get('outcome') == 'passed' for q in measured), + sum(q.get('outcome') != 'passed' for q in measured)]) + if reason or errors: + messages = [str(q.get('error') or q.get('outcome')) for q in errors] + details.append(f'- **{target} / {qid}**: {cell(str(reason or "") + " " + "; ".join(messages))[:4000]}') + lines += ['', table(['Dataset', 'Round', 'Query', 'DB', 'Status', 'Min', 'Median', 'Max', + 'Median Δ', 'Measured success', 'Measured errors'], rows), '', + '
', 'Query failures and exclusions', '', *details, '', '
'] + if not report: + lines += ['', 'Validated report unavailable; raw outcomes are shown without latency statistics.'] + return '\n'.join(lines) + + +def lifecycle(report, runs, root, targets=TARGETS): + rows = [] + for target in targets: + selected = [r for r in runs if r.get('target') == target] or [{}] + for run in selected: + load = run.get('load', {}) + ingest = load.get('ingest', {}) + if target == 'tempo' and 'ingest' not in load: + ingest = load.get('transport', {}).get('ingest', {}) + health = run.get('post_load_health', {}) + execution = run.get('execution', {}) + start, finish = run.get('started_unix_ns'), run.get('finished_unix_ns') + elapsed = (finish - start) / 1_000_000 if isinstance(start, int) and isinstance(finish, int) else None + seconds = ingest.get('wall_seconds') + rows.append([target, run.get('corpus'), run.get('load_summary', {}).get('after'), + duration(elapsed), duration(seconds * 1000 if isinstance(seconds, (int, float)) else None), + ingest.get('successful_requests'), ingest.get('failed_requests'), + execution.get('load_workers_requested'), execution.get('max_in_flight'), + execution.get('db_cpus'), execution.get('db_memory'), + health.get('running'), health.get('oom_killed'), + run.get('cleanup', {}).get('verified_absent')]) + return '\n'.join(['# Tracesbench load / lifecycle summary', '', + 'Total is the recorded target lifecycle; ingest is the sender wall time, ' + 'not the entire load/readiness phase. Health is the post-load snapshot. ' + 'Missing evidence remains blank.', '', + table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest', + 'Write success', 'Write errors', 'Load workers', 'Max in flight', + 'DB CPU', 'DB memory', 'Running after load', 'OOM after load', + 'Container removed'], rows), '', + render_resources([('dataset', root / 'runs' / 'generate')] + + [(t, root / 'runs' / t) for t in targets], table)]) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True) + parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines') + args = parser.parse_args() + targets = args.targets.split(',') + if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets): + parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo') + report, runs = load(args.root) + print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets)) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index 86713c911ed..d265801d4a0 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -26,8 +26,18 @@ on: - M default: S required: true + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -46,9 +56,9 @@ on: type: string required: false runtime_image: - description: Aliyun runtime image (tag or digest reference) + description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0 + default: auto required: true provider: description: Cloud provider for the benchmark runner @@ -89,12 +99,14 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} PROFILE: ${{ inputs.dataset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }} VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }} O11YBENCH_REF: ${{ inputs.o11ybench_ref }} - RUNTIME_IMAGE: ${{ inputs.runtime_image }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -118,6 +130,12 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" @@ -126,6 +144,16 @@ jobs: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py python3 greptimedb/tests/perf/test_ci_runner_scripts.py python3 greptimedb/tests/perf/test_agent_observability_summary.py + - name: Check runtime supports shared version-comparison data + if: inputs.compare_greptimedb + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" + docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \ + || { echo 'Comparison requires a runtime with --data-root support'; exit 1; } + - name: Validate dataset admission in runtime + run: | + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" + docker run --rm "$RUNTIME_IMAGE" scripts/logbench_agent_observability_measured.py --preflight-profile "$PROFILE" - name: Validate requested benchmark id: inputs shell: bash @@ -262,11 +290,11 @@ jobs: run: | set -euo pipefail actual= - [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]] images='{}' resolve_image() { local target=$1 reference=$2 variable=$3 metadata resolved revision - docker pull "$reference" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference" metadata=$(docker image inspect "$reference") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") if [[ "$target" == runtime ]]; then @@ -289,6 +317,9 @@ jobs: victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;; esac done + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE + fi jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \ --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \ --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \ @@ -312,6 +343,15 @@ jobs: --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json" + - name: Load and benchmark baseline GreptimeDB + id: baseline_greptimedb + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} + run: |- + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \ + --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" - name: Load and benchmark greptimedb id: greptimedb if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} @@ -357,6 +397,13 @@ jobs: GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ --owner "$OWNER" --root "$ARTIFACT_ROOT" fi + - name: Clean up baseline containers + if: always() && inputs.compare_greptimedb + run: |- + if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline" + fi - name: Benchmark summary if: always() run: | @@ -367,6 +414,16 @@ jobs: run: | python3 greptimedb/.github/scripts/agent-observability-summary.py \ --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: Baseline load / lifecycle summary + if: always() && inputs.compare_greptimedb + run: | + echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/agent-observability-summary.py \ + --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY" - name: Upload results (no dataset) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d3bad06f41d..deba5010c80 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -99,6 +99,7 @@ jobs: python3 tests/perf/test_aliyun_ecs_runner_scripts.py python3 tests/perf/test_ci_runner_scripts.py python3 tests/perf/test_agent_observability_summary.py + python3 tests/perf/test_tracesbench_summary.py check: if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }} diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml index 60081d69ad9..049faaef1a7 100644 --- a/.github/workflows/tracesbench.yml +++ b/.github/workflows/tracesbench.yml @@ -2,6 +2,36 @@ name: Tracesbench on: workflow_dispatch: inputs: + run_greptimedb: + description: Run GreptimeDB + type: boolean + default: true + required: false + run_victoriatraces: + description: Run VictoriaTraces + type: boolean + default: false + required: false + run_tempo: + description: Run Tempo + type: boolean + default: false + required: false + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false + greptimedb_tag: + description: Candidate Docker Hub greptime/greptimedb tag + type: string + default: latest + required: true dataset: description: Dataset profile (main is experimental; native queries may exceed memory limits) type: choice @@ -9,9 +39,9 @@ on: default: sanity required: true runtime_image: - description: Aliyun runtime image (tag or digest reference) + description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260928-4e52d0b0@sha256:39d2d87593788e0c47a54768f6323679b8e03d7dc72a8158a9b22cf29f883baa + default: auto required: true provider: description: Cloud provider for the benchmark runner @@ -20,7 +50,7 @@ on: default: Aliyun required: true instance_type: - description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.' + description: 'Instance type; auto = Aliyun: ecs.g9i.2xlarge, AWS: m7i.2xlarge (both 8 vCPU / 32 GiB). Enter a specific type to override.' type: string default: auto required: true @@ -45,33 +75,46 @@ on: default: 8 required: true db_cpus: - description: CPU limit per DB container + description: CPU limit per DB container (default 8 vCPU) type: string default: '8' required: true db_memory: - description: Memory limit per DB container + description: Memory limit per DB container (default 32 GiB; shared host also runs the runtime and runner) type: string - default: 16g + default: 32g + required: true + load_workers: + description: CPU workers for parallel import reading/parsing across all engines (separate from write concurrency) + type: number + default: 4 + required: true + query_timeout_seconds: + description: Timeout per query request in seconds (correctness, warmup and measurement; 1..3600) + type: number + default: 300 required: true runtime_cpus: - description: CPU limit per runtime container + description: CPU limit per runtime container (0 = unrestricted) type: string - default: '2' + default: '0' required: true runtime_memory: - description: Memory limit per runtime container (separate from DB memory) + description: Memory limit per runtime container (0 = unrestricted; separate from DB memory) type: string - default: 4g + default: '0' required: true permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} DATASET: ${{ inputs.dataset }} + GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} RUNTIME_CPUS: ${{ inputs.runtime_cpus }} RUNTIME_MEMORY: ${{ inputs.runtime_memory }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} - RUNTIME_IMAGE: ${{ inputs.runtime_image }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -79,12 +122,18 @@ env: JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }} DB_CPUS: ${{ inputs.db_cpus }} DB_MEMORY: ${{ inputs.db_memory }} + QUERY_TIMEOUT_SECONDS: ${{ inputs.query_timeout_seconds }} + LOAD_WORKERS: ${{ inputs.load_workers }} jobs: validate: runs-on: ubuntu-latest timeout-minutes: 15 outputs: instance_type: ${{ steps.runner.outputs.instance_type }} + targets: ${{ steps.inputs.outputs.targets }} + greptimedb: ${{ steps.inputs.outputs.greptimedb }} + victoriatraces: ${{ steps.inputs.outputs.victoriatraces }} + tempo: ${{ steps.inputs.outputs.tempo }} system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }} runtime_image: ${{ steps.runtime.outputs.runtime_image }} o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }} @@ -93,9 +142,44 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate selected targets + id: inputs + env: + greptimedb: ${{ inputs.run_greptimedb }} + victoriatraces: ${{ inputs.run_victoriatraces }} + tempo: ${{ inputs.run_tempo }} + run: | + set -euo pipefail + selected= + for target in greptimedb victoriatraces tempo; do + enabled=${!target} + [[ "$enabled" == true || "$enabled" == false ]] + printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT" + if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi + done + [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; } + if [[ "$greptimedb" == true ]]; then + [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; } + fi + printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT" + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner - run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" + run: | + set -euo pipefail + instance_type="$REQUESTED_INSTANCE_TYPE" + if [[ "$instance_type" == auto ]]; then + case "$PROVIDER" in + Aliyun) instance_type=ecs.g9i.2xlarge ;; + AWS) instance_type=m7i.2xlarge ;; + esac + fi + python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$instance_type" - name: Validate budgets and shared runtime id: runtime run: | @@ -115,19 +199,38 @@ jobs: ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168)) ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) [[ "$DB_CPUS" =~ ^[1-9][0-9]*$ && "$DB_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]] - [[ "$RUNTIME_CPUS" =~ ^[1-9][0-9]*$ && "$RUNTIME_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]] - [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] - docker pull "$RUNTIME_IMAGE" + [[ "$RUNTIME_CPUS" =~ ^(0|[1-9][0-9]*)$ && "$RUNTIME_MEMORY" =~ ^(0|[1-9][0-9]*[mMgG])$ ]] + [[ "$QUERY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,3}$ ]] + ((QUERY_TIMEOUT_SECONDS <= 3600)) + [[ "$LOAD_WORKERS" =~ ^[1-9][0-9]{0,2}$ ]] + ((LOAD_WORKERS <= 256)) + runtime_limits=() + [[ "$RUNTIME_CPUS" == 0 ]] || runtime_limits+=(--cpus "$RUNTIME_CPUS") + [[ "$RUNTIME_MEMORY" == 0 ]] || runtime_limits+=(--memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY") + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]] + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" metadata=$(docker image inspect "$RUNTIME_IMAGE") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \ + || { echo 'Comparison requires a runtime with --data-root support'; exit 1; } + fi revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata") [[ "$revision" =~ ^[0-9a-f]{40}$ ]] # Refuse runtimes without the trace runner and its validated dataset/report contract. - help=$(docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/run_tracesbench_docker.py --help) + help=$(docker run --rm "${runtime_limits[@]}" "$resolved" scripts/run_tracesbench_docker.py --help) + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + grep -F -- '--baseline-run' <<< "$help" \ + || { echo 'Comparison requires a runtime with --baseline-run report support'; exit 1; } + fi grep -F -- '--dataset {sanity,preflight,main}' <<< "$help" grep -F -- '--runtime-cpus' <<< "$help" grep -F -- '--runtime-memory' <<< "$help" - docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/check_run_tracesbench_docker.py + grep -F -- '--query-timeout-seconds' <<< "$help" + grep -F -- '--load-workers' <<< "$help" + grep -F -- '--targets' <<< "$help" + grep -F -- '--greptimedb-image' <<< "$help" + docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT" provision: needs: validate @@ -190,6 +293,7 @@ jobs: runs-on: ${{ needs.provision.outputs.label }} timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }} env: + TARGETS: ${{ needs.validate.outputs.targets }} RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }} RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }} RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }} @@ -224,36 +328,83 @@ jobs: available=$(df -Pk "$path" | awk 'NR == 2 {print $4}') ((available >= minimum_gib * 1024 * 1024)) || { echo "Need at least $minimum_gib GiB free on $path"; exit 1; } done - docker pull "$RESOLVED_RUNTIME_IMAGE" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE" docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json" test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF" - # Preserve the pinned catalog images and require all three targets in the report. + - name: Pull database images before generating data + run: | + set -euo pipefail + IFS=, read -ra targets <<< "$TARGETS" + for target in "${targets[@]}"; do + if [[ "$target" == greptimedb ]]; then + image="greptime/greptimedb:$GREPTIMEDB_TAG" + else + image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \ + "o11ybench/workloads/tracesbench/catalogs/$target.json") + fi + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image" + docker image inspect "$image" > "$METADATA_ROOT/$target-image.json" + if [[ "$target" == greptimedb ]]; then + resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json") + printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" + fi + done + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image" + docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json" + resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json") + printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" + fi - name: Generate selected dataset once + id: generate run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - - name: GreptimeDB load, correctness and benchmark + - name: Baseline GreptimeDB load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 \ - --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ + --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ + --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb" + - name: GreptimeDB load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb" - name: VictoriaTraces load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }} run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \ - --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Tempo load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }} run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \ - --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - - name: Validate and render report + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage report \ + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY" + - name: Validate and render report + env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + run: | + comparison_args=() + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json") + fi + python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Clean up owned containers @@ -262,6 +413,23 @@ jobs: python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Clean up baseline containers + if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ + --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Benchmark summary + if: always() + run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" + - name: Load / lifecycle summary + if: always() + run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + - name: Baseline load / lifecycle summary + if: always() && inputs.compare_greptimedb + run: | + echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/tracesbench-summary.py \ + --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | @@ -269,11 +437,15 @@ jobs: if [[ -d "$METADATA_ROOT" ]]; then rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/" fi - for entry in runs report report.json report.md; do + for entry in runs report report.json report.md version-comparison.json version-comparison.md; do if [[ -e "$RUN_ROOT/$entry" ]]; then rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/" fi done + if [[ -d "$RUN_ROOT/baseline/runs" ]]; then + mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline" + rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/" + fi - name: Upload HTML and results (no corpus) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/vmbench-long-range.yml b/.github/workflows/vmbench-long-range.yml index 357d2b21c69..aaf2dd7f437 100644 --- a/.github/workflows/vmbench-long-range.yml +++ b/.github/workflows/vmbench-long-range.yml @@ -20,8 +20,18 @@ on: - 1mo default: 1w required: true + compare_greptimedb: + description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run) + type: boolean + default: false + required: false + baseline_greptimedb_tag: + description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled + type: string + default: v1.2.1 + required: false greptimedb_tag: - description: Docker Hub greptime/greptimedb tag + description: Candidate Docker Hub greptime/greptimedb tag type: string default: latest required: true @@ -31,9 +41,9 @@ on: default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344 required: true runtime_image: - description: Aliyun runtime image (tag or digest reference) + description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29 + default: auto required: true provider: description: Cloud provider for the benchmark runner @@ -79,11 +89,13 @@ on: permissions: contents: read env: + COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} + BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} RANGE_PRESET: ${{ inputs.range_preset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }} - RUNTIME_IMAGE: ${{ inputs.runtime_image }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -107,6 +119,12 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate version comparison + env: + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} + run: | + python3 greptimedb/tests/perf/test_benchmark_version_comparison.py + python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate - name: Resolve CI runner id: runner run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE" @@ -135,8 +153,8 @@ jobs: ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168)) # Match the existing ECS workflow's provision, teardown and queue headroom. ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120)) - [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]] - docker pull "$RUNTIME_IMAGE" + [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]] + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE" metadata=$(docker image inspect "$RUNTIME_IMAGE") resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata") revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata") @@ -146,6 +164,9 @@ jobs: args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --run-classification controlled_server_benchmark --no-open-browser) + if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range") + fi # Tags become target-spec fields; reject separators before passing them to the planner. for target in greptimedb victoriametrics; do case "$target" in @@ -273,13 +294,13 @@ jobs: available=$(df -Pk "$path" | awk 'NR == 2 {print $4}') ((available >= 10 * 1024 * 1024)) || { echo "Need at least 10 GiB free on $path"; exit 1; } done - docker pull "$RESOLVED_RUNTIME_IMAGE" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE" docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json" test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF" plan="$METADATA_ROOT/warm.json" while IFS=$'\t' read -r target reference; do echo "Resolving $target: $reference" - docker pull "$reference" + bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference" docker image inspect "$reference" > "$METADATA_ROOT/$target-image.json" digest=$(jq -er '.[0].RepoDigests[0]' "$METADATA_ROOT/$target-image.json") jq --arg target "$target" --arg image "$digest" '.targets |= map(if .name == $target then .image = $image else . end)' "$plan" > "$plan.tmp" @@ -303,6 +324,10 @@ jobs: if: always() run: | docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv + - name: GreptimeDB version comparison + if: always() && inputs.compare_greptimedb + run: | + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | diff --git a/tests/perf/test_agent_observability_summary.py b/tests/perf/test_agent_observability_summary.py index 8223231c908..a6c1774ba0f 100644 --- a/tests/perf/test_agent_observability_summary.py +++ b/tests/perf/test_agent_observability_summary.py @@ -14,11 +14,13 @@ import importlib.util import json +import sys import tempfile import unittest from pathlib import Path SCRIPT = Path(__file__).parents[2] / '.github/scripts/agent-observability-summary.py' +sys.path.insert(0, str(SCRIPT.parent)) spec = importlib.util.spec_from_file_location('observability_summary', SCRIPT) summary = importlib.util.module_from_spec(spec) spec.loader.exec_module(summary) diff --git a/tests/perf/test_benchmark_version_comparison.py b/tests/perf/test_benchmark_version_comparison.py new file mode 100644 index 00000000000..56c2619e81c --- /dev/null +++ b/tests/perf/test_benchmark_version_comparison.py @@ -0,0 +1,138 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +SCRIPTS = Path(__file__).resolve().parents[2] / '.github/scripts' +spec = importlib.util.spec_from_file_location('comparison', SCRIPTS / 'benchmark-version-comparison.py') +comparison = importlib.util.module_from_spec(spec) +spec.loader.exec_module(comparison) + + +def write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value)) + + +class VersionComparison(unittest.TestCase): + def test_opt_in_validation(self): + comparison.validate({}) + comparison.validate({'COMPARE_GREPTIMEDB': 'false', 'GREPTIMEDB_TAG': 'invalid\n'}) + good = dict(COMPARE_GREPTIMEDB='true', RUN_GREPTIMEDB='true', + BASELINE_GREPTIMEDB_TAG='v1.2.1', GREPTIMEDB_TAG='v1.3.0-beta.1') + comparison.validate(good) + for key, value in [('RUN_GREPTIMEDB', 'false'), ('BASELINE_GREPTIMEDB_TAG', ''), + ('BASELINE_GREPTIMEDB_TAG', 'v1.3.0-beta.1'), + ('BASELINE_GREPTIMEDB_TAG', 'a,image=other'), ('GREPTIMEDB_TAG', 'x\ny'), + ('COMPARE_GREPTIMEDB', 'yes')]: + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + comparison.validate(good | {key: value}) + + def test_improvement_direction(self): + self.assertAlmostEqual(comparison.improvement(100, 80), 20) + self.assertAlmostEqual(comparison.improvement(100, 120), -20) + self.assertAlmostEqual(comparison.improvement(100, 120, True), 20) + for old, new in [(0, 3), (None, 4), (1, -1), (1, float('nan')), (True, 2)]: + self.assertIsNone(comparison.improvement(old, new)) + + def test_agent_result_gates(self): + summary = dict(passed=True, contract_sha256='contract', data_profile='S', query_window=[1, 2], + warmup_runs=1, measured_runs=5, per_query=[dict(query_id='Q01', result_fingerprint='answer', + success=5, errors=0, timeouts=0, p50_ms=100, p95_ms=100, p99_ms=100)]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + base = root / 'baseline/greptimedb-1/greptimedb-1/measured-summary.json' + candidate = root / 'greptimedb-1/greptimedb-1/measured-summary.json' + for r in (root / 'baseline', root): + (r / 'greptimedb-1').mkdir(parents=True, exist_ok=True) + (r / 'greptimedb-1/exit-code.txt').write_text('0\n') + write(base, summary) + changed = copy.deepcopy(summary); changed['per_query'][0]['p50_ms'] = 80 + write(candidate, changed) + self.assertAlmostEqual(comparison.agent(root)[0]['improvement_percent'], 20) + for key, value in [('errors', 1), ('timeouts', 1), ('result_fingerprint', 'wrong'), ('success', 0)]: + broken = copy.deepcopy(changed); broken['per_query'][0][key] = value + write(candidate, broken) + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + (root / 'greptimedb-1/exit-code.txt').write_text('1\n') + write(candidate, changed) + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + (root / 'greptimedb-1/exit-code.txt').write_text('0\n') + changed['query_window'] = [2, 3]; write(candidate, changed) + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + candidate.unlink() + self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root))) + + def test_traces_uses_measured_only_and_rejects_failed_query(self): + answer = dict(query_id='T01', outcome='passed', fingerprint='answer', row_count=1, elapsed_ms=100) + sample = dict(answer, phase='measured') + record = dict(success=True, expected={'contracts': {'T01': {'fingerprint': 'answer', 'row_count': 1}}}, + protocol={'runs': 5}, audit_expected={}, corpus='sanity', + execution=dict(db_cpus='8', db_memory='16g', load_workers_requested=4, max_in_flight=8), + correctness=[answer], samples=[dict(sample, phase='warmup', elapsed_ms=99999), sample, sample]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp);base = root / 'baseline/runs/greptimedb/run.json'; candidate = root / 'runs/greptimedb/run.json' + write(base, record); changed = copy.deepcopy(record) + for s in changed['samples']: + if s['phase'] == 'measured': s['elapsed_ms'] = 80 + write(candidate, changed) + self.assertAlmostEqual(comparison.traces(root)[0]['improvement_percent'], 20) + self.assertIsNone(comparison.traces(root)[1]['improvement_percent']) + for key, value in [('outcome', 'failed'), ('fingerprint', 'wrong'), ('row_count', 2)]: + broken = copy.deepcopy(changed); broken['samples'][1][key] = value; write(candidate, broken) + self.assertIsNone(comparison.traces(root)[0]['improvement_percent']) + + def test_long_range_gates(self): + summary = dict(workload='vmbench-long-range', query_file_sha256='hash', start='start', end='end', step='30m', + profile='small', data_duration='32d', scrape_interval='30s', cache_policy_effective='warm', + mode='serial', connection_mode='force-close', request_timeout_seconds=120, tries=5, warmup_runs=1, + errors=0, timeouts=0, per_query=[dict(query_index=1, query='up', series=1, points=2, + success=5, errors=0, timeouts=0, p50_ms=100)]) + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); base = root/'greptimedb-baseline/results-vmbench-long-range/serial-summary.json' + candidate = root/'greptimedb/results-vmbench-long-range/serial-summary.json' + health = {phase: {target: {'state': 'Running=true ExitCode=0 OOMKilled=false'} + for target in ('greptimedb-baseline', 'greptimedb')} + for phase in ('post_load_results', 'post_query_results')} + health['post_load_results'] = {k: {'docker': v} for k, v in health['post_load_results'].items()} + write(root / 'run-manifest.json', health) + write(base, summary); write(candidate, summary) + self.assertEqual(comparison.long_range(root)[0]['improvement_percent'], 0) + for key, value in [('start','different'),('tries',1),('query_file_sha256','other'),('errors',1)]: + write(candidate, summary | {key:value}) + self.assertIsNone(comparison.long_range(root)[0]['improvement_percent']) + write(candidate, summary) + broken=copy.deepcopy(health);broken['post_query_results']['greptimedb']['state']='Running=false ExitCode=137 OOMKilled=true' + write(root / 'run-manifest.json', broken) + self.assertIsNone(comparison.long_range(root)[0]['improvement_percent']) + write(root / 'run-manifest.json', health) + empty=copy.deepcopy(summary);empty['per_query'][0]['points']=0;write(candidate,empty) + self.assertIsNone(comparison.long_range(root)[0]['improvement_percent']) + + def test_missing_results_and_rendering(self): + with tempfile.TemporaryDirectory() as tmp: + for load in (comparison.agent, comparison.traces, comparison.long_range): + self.assertTrue(all(r['improvement_percent'] is None for r in load(Path(tmp)))) + rendered = comparison.render(dict(baseline_tag='old', candidate_tag='', rows=[])) + self.assertIn('<new|tag>', rendered) + self.assertIn('Results unavailable', rendered) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/perf/test_ci_runner_scripts.py b/tests/perf/test_ci_runner_scripts.py index f9742079b49..c703e41c62b 100644 --- a/tests/perf/test_ci_runner_scripts.py +++ b/tests/perf/test_ci_runner_scripts.py @@ -16,6 +16,7 @@ """Offline regression coverage for CI runner selection and lifecycle.""" import importlib.util +import json import os import re import subprocess @@ -77,6 +78,67 @@ class ProviderConfigTest(unittest.TestCase): with self.subTest(provider=provider, requested=requested): self.assertEqual(config.resolve_instance_type(provider, requested), expected) + def test_traces_workflow_runner_defaults_and_overrides(self): + workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text() + step = workflow.split(" - name: Resolve CI runner\n", 1)[1].split(" - name:", 1)[0] + command = textwrap.dedent(step.split(" run: |\n", 1)[1]) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "greptimedb").symlink_to(SCRIPTS.parents[1], target_is_directory=True) + output = root / "output" + for provider, requested, expected in ( + ("Aliyun", "auto", "ecs.g9i.2xlarge"), + ("AWS", "auto", "m7i.2xlarge"), + ("Aliyun", "ecs.c9i.4xlarge", "ecs.c9i.4xlarge"), + ("AWS", "c7i.4xlarge", "c7i.4xlarge"), + ("invalid", "auto", None), + ("AWS", "ecs.g9i.2xlarge", None), + ): + with self.subTest(provider=provider, requested=requested): + output.write_text("") + result = subprocess.run( + ["bash", "-c", command], cwd=root, capture_output=True, text=True, + env=dict(os.environ, PROVIDER=provider, REQUESTED_INSTANCE_TYPE=requested, + GITHUB_OUTPUT=str(output)), timeout=10, + ) + if expected is None: + self.assertNotEqual(result.returncode, 0) + self.assertEqual(output.read_text(), "") + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), f"instance_type={expected}\n") + self.assertRegex(workflow, r"(?s)db_cpus:.*?default: '8'") + self.assertRegex(workflow, r"(?s)db_memory:.*?default: 32g") + + def test_traces_selected_targets_and_image_tag(self): + workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text() + step = workflow.split(" - name: Validate selected targets\n", 1)[1].split(" - name:", 1)[0] + command = textwrap.dedent(step.split(" run: |\n", 1)[1]) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + for enabled, tag, expected in ( + (("true", "false", "false"), "latest", "greptimedb"), + (("false", "true", "false"), "", "victoriatraces"), + (("false", "false", "true"), "", "tempo"), + (("true", "true", "true"), "v1.3.0-beta.1", "greptimedb,victoriatraces,tempo"), + (("false", "false", "false"), "latest", None), + (("true", "false", "false"), "latest; touch /tmp/unexpected", None), + (("true", "false", "false"), "", None), + (("invalid", "false", "false"), "latest", None), + ): + with self.subTest(enabled=enabled, tag=tag): + output.write_text("") + env = dict(os.environ, GREPTIMEDB_TAG=tag, GITHUB_OUTPUT=str(output)) + env.update(zip(("greptimedb", "victoriatraces", "tempo"), enabled)) + result = subprocess.run(["bash", "-c", command], env=env, + capture_output=True, text=True, timeout=5) + if expected is None: + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("targets=", output.read_text()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"targets={expected}\n", output.read_text()) + def test_wrong_provider_and_shell_fragments_rejected(self): for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"), ("Aliyun", "c7i.2xlarge"), ("AWS", ""), @@ -401,6 +463,154 @@ class AwsTeardownTest(unittest.TestCase): self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1) self.assertEqual([c.args[1] for c in delete.call_args_list], ["i-a", "i-b"]) +class BenchmarkImagePullTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.env = dict(os.environ, PATH=f"{self.root}:{os.environ['PATH']}", + PULL_TEST_ROOT=str(self.root)) + self.tool("docker", ''' + echo "$*" >> "$PULL_TEST_ROOT/docker.log" + count=$(wc -l < "$PULL_TEST_ROOT/docker.log") + codes=${PULL_TEST_CODES:-0} + code=$(printf '%s' "$codes" | cut -d, -f"$count") + exit "${code:-0}" + ''') + self.tool("timeout", ''' + echo "$*" >> "$PULL_TEST_ROOT/timeout.log" + shift 2 + if [[ ${PULL_TEST_TIMEOUT:-0} == 1 ]]; then exit 124; fi + exec "$@" + ''') + self.tool("sleep", 'echo "$*" >> "$PULL_TEST_ROOT/sleep.log"') + + def tool(self, name, body): + p = self.root / name + p.write_text("#!/bin/bash\nset -eu\n" + textwrap.dedent(body)) + p.chmod(0o755) + + def lines(self, name): + p = self.root / (name + ".log") + return p.read_text().splitlines() if p.exists() else [] + + def pull(self, codes, *args): + self.env["PULL_TEST_CODES"] = codes + return subprocess.run(["bash", str(SCRIPTS / "pull-benchmark-image.sh"), *args], + env=self.env, capture_output=True, text=True, timeout=5) + + def test_success_does_not_retry(self): + result = self.pull("0", "registry/test@sha256:" + "a" * 64) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(self.lines("docker")), 1) + self.assertEqual(self.lines("sleep"), []) + self.assertTrue(self.lines("timeout")[0].startswith("--kill-after=10s 180s docker pull ")) + + def test_transient_failure_then_success(self): + result = self.pull("1,1,0", "registry/test:tag") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.lines("docker"), ["pull registry/test:tag"] * 3) + self.assertEqual(self.lines("sleep"), ["10", "20"]) + self.assertIn("attempt 3/3", result.stderr) + + def test_exhaustion_preserves_failure(self): + result = self.pull("1,1,7", "registry/test:tag") + self.assertEqual(result.returncode, 7) + self.assertEqual(len(self.lines("docker")), 3) + self.assertEqual(self.lines("sleep"), ["10", "20"]) + + def test_timeout_is_bounded_and_fatal(self): + self.env["PULL_TEST_TIMEOUT"] = "1" + result = self.pull("0", "registry/test:tag") + self.assertEqual(result.returncode, 124) + self.assertEqual(len(self.lines("timeout")), 3) + self.assertEqual(self.lines("sleep"), ["10", "20"]) + + def test_bad_arguments_do_not_call_docker(self): + for args in ([], [""], ["--help"], ["a", "b"]): + with self.subTest(args=args): + self.assertEqual(self.pull("0", *args).returncode, 2) + self.assertEqual(self.lines("docker"), []) + + def test_traces_prefetch_executes_catalog_refs_and_stops_on_failure(self): + traces = (SCRIPTS.parent / 'workflows/tracesbench.yml').read_text() + start = traces.index(' - name: Pull database images before generating data') + end = traces.index(' - name: Generate selected dataset once') + command = textwrap.dedent(traces[start:end].split(' run: |\n', 1)[1]) + helper = self.root / 'greptimedb/.github/scripts/pull-benchmark-image.sh' + helper.parent.mkdir(parents=True) + helper.write_text((SCRIPTS / helper.name).read_text()) + catalogs = self.root / 'o11ybench/workloads/tracesbench/catalogs' + catalogs.mkdir(parents=True) + images = [f'registry/{name}@sha256:' + 'a' * 64 + for name in ('greptimedb', 'victoriatraces', 'tempo')] + for name, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images): + (catalogs / f'{name}.json').write_text(json.dumps({'target_identity': {'image': image}})) + self.tool('docker', ''' + echo "$*" >> "$PULL_TEST_ROOT/docker.log" + if [[ $1 == pull && $2 == ${PULL_FAIL_IMAGE:-} ]]; then exit 1; fi + if [[ $1 == image ]]; then printf '[{"RepoDigests":["greptime/greptimedb@sha256:%s"]}]\\n' "$(printf 'a%.0s' {1..64})"; fi + ''') + images[0] = 'greptime/greptimedb:v1.3.0-beta.1' + self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1', + GITHUB_ENV=str(self.root / 'env'), BASELINE_GREPTIMEDB_TAG='v1.2.1') + for selected, compare in ((('greptimedb',), False), (('tempo',), False), + (('greptimedb', 'victoriatraces', 'tempo'), False), + (('greptimedb',), True), (('greptimedb', 'victoriatraces', 'tempo'), True)): + with self.subTest(selected=selected): + self.env['TARGETS'] = ','.join(selected) + self.env['COMPARE_GREPTIMEDB'] = str(compare).lower() + (self.root / 'env').write_text('') + (self.root / 'docker.log').unlink(missing_ok=True) + result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env, + capture_output=True, text=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images) + if target in selected] + if compare: + selected_images.append('greptime/greptimedb:v1.2.1') + self.assertEqual(self.lines('docker'), + [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')]) + expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n' + expected_env = expected_env if 'greptimedb' in selected else '' + if compare: + expected_env += 'BASELINE_GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n' + self.assertEqual((self.root / 'env').read_text(), expected_env) + (self.root / 'docker.log').unlink() + self.env['PULL_FAIL_IMAGE'] = images[1] + result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env, + capture_output=True, text=True, timeout=5) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(self.lines('docker'), [f'pull {images[0]}', f'image inspect {images[0]}'] + + [f'pull {images[1]}'] * 3) + self.assertNotIn(f'pull {images[2]}', self.lines('docker')) + + def test_workflows_pull_before_generation(self): + workflows = SCRIPTS.parent / "workflows" + agent = (workflows / "agent-observability.yml").read_text() + traces = (workflows / "tracesbench.yml").read_text() + long_range = (workflows / "vmbench-long-range.yml").read_text() + for content in (agent, traces, long_range): + self.assertNotIn("docker pull ", content) + for reference in ("RUNTIME_IMAGE", "RESOLVED_RUNTIME_IMAGE", "reference"): + self.assertIn(f'pull-benchmark-image.sh "${reference}"', long_range) + self.assertLess(long_range.index('pull-benchmark-image.sh "$reference"'), + long_range.index('- name: Warm serial and concurrent benchmarks')) + self.assertIn('pull-benchmark-image.sh "$reference"', agent) + self.assertLess(agent.index('resolve_image runtime'), agent.index('- name: Generate dataset once')) + self.assertIn('pull-benchmark-image.sh "$RUNTIME_IMAGE"', traces) + self.assertIn('pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"', traces) + start = traces.index(' - name: Pull database images before generating data') + end = traces.index(' - name: Generate selected dataset once') + self.assertLess(start, end) + step = traces[start:end] + self.assertIn('IFS=, read -ra targets <<< "$TARGETS"', step) + self.assertIn('for target in "${targets[@]}"', step) + self.assertIn('o11ybench/workloads/tracesbench/catalogs/$target.json', step) + self.assertIn('pull-benchmark-image.sh "$image"', step) + self.assertIn('set -euo pipefail', step) + self.assertNotIn('continue-on-error', step) + if __name__ == "__main__": unittest.main() diff --git a/tests/perf/test_tracesbench_summary.py b/tests/perf/test_tracesbench_summary.py new file mode 100644 index 00000000000..cb7c4d8e06e --- /dev/null +++ b/tests/perf/test_tracesbench_summary.py @@ -0,0 +1,54 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).parents[2] / '.github/scripts/tracesbench-summary.py' +sys.path.insert(0, str(SCRIPT.parent)) +spec = importlib.util.spec_from_file_location('tracesbench_summary', SCRIPT) +summary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(summary) + + +class LifecycleTest(unittest.TestCase): + def test_ingestion_layouts(self): + metrics = {'wall_seconds': 0.121723417, 'successful_requests': 17, 'failed_requests': 0} + cases = [ + (target, {'ingest': metrics}, ['121.7 ms', '17', '0']) + for target in summary.TARGETS + ] + [ + ('tempo', {'transport': {'ingest': metrics}}, ['121.7 ms', '17', '0']), + ('tempo', {'ingest': {'wall_seconds': 0, 'successful_requests': 0, 'failed_requests': 0}, + 'transport': {'ingest': metrics}}, ['0 µs', '0', '0']), + ('tempo', {'ingest': {}, 'transport': {'ingest': metrics}}, ['—', '—', '—']), + ('tempo', {'transport': {}}, ['—', '—', '—']), + ] + [(target, {}, ['—', '—', '—']) for target in summary.TARGETS] + [ + (target, {'transport': {'ingest': metrics}}, ['—', '—', '—']) + for target in ('greptimedb', 'victoriatraces') + ] + with tempfile.TemporaryDirectory() as tmp: + for target, load, expected in cases: + with self.subTest(target=target, load=load): + text = summary.lifecycle({}, [{'target': target, 'load': load}], Path(tmp), [target]) + row = next(line for line in text.splitlines() if line.startswith('| ' + target + ' |')) + cells = [cell.strip() for cell in row.split('|')[1:-1]] + self.assertEqual(cells[4:7], expected) + + +if __name__ == '__main__': + unittest.main()