diff --git a/.github/scripts/agent-observability-summary.py b/.github/scripts/agent-observability-summary.py
index 38ba4e9dbdd..33ba3ad7e6e 100644
--- a/.github/scripts/agent-observability-summary.py
+++ b/.github/scripts/agent-observability-summary.py
@@ -18,6 +18,8 @@ import json
import math
from pathlib import Path
+from benchmark_resource_summary import render_resources
+
TARGETS = ('greptimedb', 'clickhouse', 'victorialogs')
@@ -201,6 +203,7 @@ def lifecycle(root, targets):
rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else
'invalid timestamp units (legacy artifact)', entry.get('exit_code')])
lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), '']
+ lines.append(render_resources([('dataset', root)] + [(t, root / f'{t}-1') for t in targets], table))
return '\n'.join(lines)
diff --git a/.github/scripts/benchmark-version-comparison.py b/.github/scripts/benchmark-version-comparison.py
new file mode 100644
index 00000000000..1aa744df9ad
--- /dev/null
+++ b/.github/scripts/benchmark-version-comparison.py
@@ -0,0 +1,191 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas."""
+import argparse
+import html
+import json
+import math
+import os
+from pathlib import Path
+import re
+import statistics
+
+
+def read(path):
+ return json.loads(path.read_text()) if path.is_file() else {}
+
+
+def validate(env):
+ enabled = env.get('COMPARE_GREPTIMEDB', 'false')
+ if enabled not in ('true', 'false'):
+ raise ValueError('COMPARE_GREPTIMEDB must be true or false')
+ if enabled == 'false':
+ return
+ if env.get('RUN_GREPTIMEDB') != 'true':
+ raise ValueError('Version comparison requires Run GreptimeDB')
+ for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'):
+ if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')):
+ raise ValueError(f'Invalid {key}')
+ if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']:
+ raise ValueError('Baseline and candidate must have different image tags')
+
+
+def same(a, b, fields):
+ return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields)
+
+
+def finite(value):
+ return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def improvement(old, new, throughput=False):
+ if not finite(old) or not finite(new) or old <= 0:
+ return None
+ return (new / old - 1 if throughput else 1 - new / old) * 100
+
+
+def row(query, metric, old, new, compatible, reason):
+ delta = improvement(old, new, metric == 'QPS') if compatible else None
+ return dict(query=query, metric=metric, baseline=old, candidate=new,
+ improvement_percent=delta, status='comparable' if delta is not None else reason)
+
+
+def agent(root):
+ def load(r):
+ return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json')
+ a, b = load(root / 'baseline'), load(root)
+ finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)]
+ compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished)
+ and a.get('passed') is True and b.get('passed') is True
+ and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs')))
+ qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b))
+ rows = []
+ for q in sorted(qa.keys() | qb.keys()):
+ old, new = qa.get(q, {}), qb.get(q, {})
+ ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0
+ ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+ for key in ('p50_ms', 'p95_ms', 'p99_ms'):
+ rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+ return rows
+
+
+def traces(root):
+ a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root))
+ compatible = (a.get('success') is True and b.get('success') is True
+ and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus'))
+ and same(a.get('execution', {}), b.get('execution', {}),
+ ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight')))
+ rows = []
+ for q in (f'T{i:02}' for i in range(1, 9)):
+ values, identities, counts = [], [], []
+ ok = compatible
+ for run in (a, b):
+ correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q]
+ all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q]
+ measured = [s for s in all_samples if s.get('phase') == 'measured']
+ passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples)
+ expected = run.get('expected', {}).get('contracts', {}).get(q, {})
+ passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None
+ and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples)
+ elapsed = [s.get('elapsed_ms') for s in measured]
+ passed = passed and all(finite(v) for v in elapsed)
+ ok = ok and passed
+ values.append(statistics.median(elapsed) if passed else None)
+ identities.append(expected)
+ counts.append((len(correctness), len(all_samples), len(measured)))
+ ok = ok and identities[0] == identities[1] and counts[0] == counts[1]
+ rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result'))
+ return rows
+
+
+def long_range(root):
+ rows = []
+ manifest = read(root / 'run-manifest.json')
+ healthy = all(
+ all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results'
+ else manifest.get(phase, {}).get(target, {})).get('state', '').split()
+ for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false'))
+ for phase in ('post_load_results', 'post_query_results')
+ for target in ('greptimedb-baseline', 'greptimedb'))
+ for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'):
+ a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json'))
+ for name in ('greptimedb-baseline', 'greptimedb'))
+ fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration',
+ 'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds')
+ fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds')
+ compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b))
+ qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b))
+ for q in sorted(qa.keys() | qb.keys()):
+ old, new = qa.get(q, {}), qb.get(q, {})
+ ok = compatible and same(old, new, ('query', 'series', 'points'))
+ ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0
+ and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+ for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',):
+ rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key,
+ old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+ return rows
+
+
+def render(document):
+ def cell(value):
+ return html.escape(str(value)).replace('|', '|').replace('\n', ' ')
+ def value(v, metric):
+ if not finite(v):
+ return '—'
+ if metric == 'QPS':
+ return f'{v:.3f}'
+ return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms'
+ lines = ['# GreptimeDB version comparison', '',
+ f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '',
+ 'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; '
+ 'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. '
+ 'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '',
+ '| Query | Metric | Baseline | Candidate | Improvement | Status |',
+ '| --- | --- | --- | --- | --- | --- |']
+ for r in document['rows']:
+ delta = r['improvement_percent']
+ lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']),
+ value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |')
+ if document.get('kind') == 'long-range':
+ lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.']
+ if not document['rows']:
+ lines += ['', 'Results unavailable; comparison not performed.']
+ return '\n'.join(lines) + '\n'
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('command', choices=('validate', 'report'))
+ parser.add_argument('--kind', choices=('agent', 'traces', 'long-range'))
+ parser.add_argument('--root', type=Path)
+ args = parser.parse_args()
+ if args.command == 'validate':
+ validate(os.environ)
+ return
+ if args.root is None or args.kind is None:
+ parser.error('report requires --root and --kind')
+ document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''),
+ 'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''),
+ 'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)}
+ text = render(document)
+ # Preserve partial comparisons even if a target failed before creating output.
+ args.root.mkdir(parents=True, exist_ok=True)
+ (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n')
+ (args.root / 'version-comparison.md').write_text(text)
+ print(text, end='')
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.github/scripts/benchmark_resource_summary.py b/.github/scripts/benchmark_resource_summary.py
new file mode 100644
index 00000000000..3a0a896c46e
--- /dev/null
+++ b/.github/scripts/benchmark_resource_summary.py
@@ -0,0 +1,99 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Render shared O11yBench sampled resource artifacts for lifecycle summaries."""
+import json
+import math
+
+
+def number(value):
+ return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def metric(values, key, statistic):
+ entry = values.get(key, {})
+ value = entry.get(statistic)
+ return value if number(entry.get('samples')) and entry['samples'] > 0 and number(value) else None
+
+
+def decimal(value, suffix=''):
+ return f'{value:.2f}{suffix}' if number(value) else '—'
+
+
+def memory(value):
+ if not number(value):
+ return '—'
+ for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
+ if value < 1024 or unit == 'TiB':
+ return decimal(value, f' {unit}')
+ value /= 1024
+
+
+def capacities(path):
+ # Quotas are metadata, not recomputed CPU statistics. Bound reads on long runs.
+ result = {}
+ try:
+ with path.open() as source:
+ for index, line in enumerate(source):
+ if index == 256:
+ break
+ record = json.loads(line)
+ cores = record.get('host', {}).get('cpu_count')
+ if number(cores) and cores > 0:
+ result.setdefault('host', cores)
+ for role, values in record.get('containers', {}).items():
+ quota = values.get('configured_cpu_limit_cores')
+ if number(quota):
+ result.setdefault(role, quota)
+ if all(role in result for role in ('host', 'runtime', 'db')):
+ break
+ except (OSError, ValueError, TypeError, AttributeError):
+ pass
+ return result
+
+
+def render_resources(scopes, table):
+ rows = []
+ for scope, directory in scopes:
+ try:
+ summary = json.loads((directory / 'resources.summary.json').read_text())
+ phases = summary.get('phases', {}) if summary.get('schema') == 'o11ybench-resources-v1' else {}
+ except (OSError, ValueError, TypeError, AttributeError):
+ phases = {}
+ if not isinstance(phases, dict) or not phases:
+ rows.append([scope, 'unavailable', *['—'] * 9])
+ continue
+ limits = capacities(directory / 'resources.jsonl')
+ for phase, data in phases.items():
+ for role in ('host', 'runtime', 'db'):
+ values = data.get('host', {}) if role == 'host' else data.get('containers', {}).get(role, {})
+ if role == 'db' and scope == 'dataset':
+ continue
+ quota = limits.get(role)
+ cpu_samples = values.get('cpu_used_cores', {}).get('samples')
+ rows.append([scope, phase, role, cpu_samples,
+ decimal(metric(values, 'cpu_used_cores', 'mean')),
+ decimal(metric(values, 'cpu_used_cores', 'max')),
+ 'unlimited' if quota == 0 else decimal(quota),
+ memory(metric(values, 'memory_used_bytes' if role == 'host' else 'memory_usage_bytes', 'max')),
+ decimal(metric(values, 'io_wait_percent', 'mean'), '%'),
+ decimal(metric(values, 'io_wait_percent', 'max'), '%'),
+ memory(metric(values, 'swap_used_bytes', 'max'))])
+ return '\n'.join(['## Sampled resources', '',
+ 'CPU is measured in cores. Mean/max summarize available samples, not continuous peaks; '
+ 'short phases may have no CPU sample. CPU capacity is host cores or the configured container quota. '
+ 'Host memory is total minus available; Docker memory excludes cache where supported. '
+ 'Swap is host-wide usage, not swap activity. Missing evidence stays —.', '',
+ table(['Scope', 'Phase', 'Component', 'CPU samples', 'Mean CPU cores', 'Max CPU cores',
+ 'CPU capacity', 'Max memory', 'Mean I/O wait', 'Max I/O wait', 'Max swap'], rows), ''])
diff --git a/.github/scripts/pull-benchmark-image.sh b/.github/scripts/pull-benchmark-image.sh
new file mode 100644
index 00000000000..1a26a4aeaee
--- /dev/null
+++ b/.github/scripts/pull-benchmark-image.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Pull only: never retry database writes, benchmarks, or cloud provisioning.
+set -euo pipefail
+
+if [[ $# != 1 || -z "$1" || "$1" == -* ]]; then
+ echo "Usage: $0 IMAGE" >&2
+ exit 2
+fi
+image=$1
+attempts=3
+for ((attempt=1; attempt<=attempts; attempt++)); do
+ echo "Pulling $image (attempt $attempt/$attempts)" >&2
+ if timeout --kill-after=10s 180s docker pull "$image"; then
+ exit 0
+ else
+ status=$?
+ fi
+ if ((attempt == attempts)); then
+ echo "Failed to pull $image after $attempts attempts (exit $status)" >&2
+ exit "$status"
+ fi
+ delay=$((attempt * 10))
+ echo "Pull failed (exit $status); retrying in ${delay}s" >&2
+ sleep "$delay"
+done
diff --git a/.github/scripts/tracesbench-summary.py b/.github/scripts/tracesbench-summary.py
new file mode 100644
index 00000000000..087a67498f4
--- /dev/null
+++ b/.github/scripts/tracesbench-summary.py
@@ -0,0 +1,169 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import html
+import math
+from pathlib import Path
+
+from benchmark_resource_summary import render_resources
+
+TARGETS = ('greptimedb', 'victoriatraces', 'tempo')
+
+
+def duration(ms):
+ if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0:
+ return '—'
+ if ms < 1:
+ return f'{ms * 1000:.0f} µs'
+ if ms < 1000:
+ return f'{ms:.1f} ms'
+ if ms < 60_000:
+ return f'{ms / 1000:.2f} s'
+ minutes, seconds = divmod(round(ms / 1000), 60)
+ hours, minutes = divmod(minutes, 60)
+ return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s'
+
+
+def cell(value):
+ return html.escape(str(value if value is not None else '—')).replace('|', '|').replace('\n', ' ')
+
+
+def read(path):
+ if not path.exists():
+ return {}
+ return json.loads(path.read_text())
+
+
+def table(headers, rows):
+ return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |',
+ '| ' + ' | '.join('---' for _ in headers) + ' |'] +
+ ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows])
+
+
+def delta(value, baseline):
+ if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)):
+ return '—'
+ return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—'
+
+
+
+def load(root):
+ report = read(root / 'report.json')
+ runs = [read(p) for p in sorted((root / 'runs').glob('*/run.json'))]
+ return report, runs
+
+
+def benchmark(report, runs, targets=TARGETS):
+ lines = ['# Tracesbench benchmark summary', '',
+ 'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. '
+ 'Min/median/max come from the validated report, not recalculated samples. '
+ 'Counts exclude warmup. Missing or excluded results have no latency comparison.',
+ '', f'Classification: {cell(report.get("classification"))}. '
+ 'See the uploaded HTML for query semantics and client-side compensation.', '']
+ statistics = {}
+ for corpus in report.get('per_corpus', []):
+ for stat in corpus.get('statistics', []):
+ if stat['target'] not in targets:
+ continue
+ statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat
+ rows, details = [], []
+ for target in targets:
+ selected = [run for run in runs if run.get('target') == target]
+ if not selected:
+ lines.append(f'- **{target}**: unavailable / not run.')
+ for run in selected:
+ lines.append(f'- **{target}**: lifecycle '
+ f'{"completed" if run.get("success") is True else "failed / incomplete"}; '
+ f'query failures: {len(run.get("query_failures", []))}.')
+ for qid in (f'T{i:02}' for i in range(1, 9)):
+ correct = [q for q in run.get('correctness', []) if q.get('query_id') == qid]
+ measured = [q for q in run.get('samples', [])
+ if q.get('query_id') == qid and q.get('phase') == 'measured']
+ errors = [q for q in correct + run.get('samples', [])
+ if q.get('query_id') == qid and q.get('outcome') not in ('passed', 'not_executed')]
+ reason = (run.get('exclusions', {}).get(qid) or
+ run.get('execution', {}).get('query_skips', {}).get(qid))
+ stat = statistics.get((run.get('corpus'), run.get('round'), target, qid), {})
+ base = statistics.get((run.get('corpus'), run.get('round'), 'greptimedb', qid), {})
+ status = ('failed' if errors else 'excluded' if reason else
+ 'passed' if stat else
+ 'correctness passed; report unavailable' if correct and
+ all(q.get('outcome') == 'passed' for q in correct) else 'unavailable / incomplete')
+ rows.append([run.get('corpus'), run.get('round'), qid, target, status,
+ duration(stat.get('min_ms')), duration(stat.get('median_ms')),
+ duration(stat.get('max_ms')),
+ delta(stat.get('median_ms'), base.get('median_ms')) if target != 'greptimedb' else '—',
+ sum(q.get('outcome') == 'passed' for q in measured),
+ sum(q.get('outcome') != 'passed' for q in measured)])
+ if reason or errors:
+ messages = [str(q.get('error') or q.get('outcome')) for q in errors]
+ details.append(f'- **{target} / {qid}**: {cell(str(reason or "") + " " + "; ".join(messages))[:4000]}')
+ lines += ['', table(['Dataset', 'Round', 'Query', 'DB', 'Status', 'Min', 'Median', 'Max',
+ 'Median Δ', 'Measured success', 'Measured errors'], rows), '',
+ '', 'Query failures and exclusions
', '', *details, '', ' ']
+ if not report:
+ lines += ['', 'Validated report unavailable; raw outcomes are shown without latency statistics.']
+ return '\n'.join(lines)
+
+
+def lifecycle(report, runs, root, targets=TARGETS):
+ rows = []
+ for target in targets:
+ selected = [r for r in runs if r.get('target') == target] or [{}]
+ for run in selected:
+ load = run.get('load', {})
+ ingest = load.get('ingest', {})
+ if target == 'tempo' and 'ingest' not in load:
+ ingest = load.get('transport', {}).get('ingest', {})
+ health = run.get('post_load_health', {})
+ execution = run.get('execution', {})
+ start, finish = run.get('started_unix_ns'), run.get('finished_unix_ns')
+ elapsed = (finish - start) / 1_000_000 if isinstance(start, int) and isinstance(finish, int) else None
+ seconds = ingest.get('wall_seconds')
+ rows.append([target, run.get('corpus'), run.get('load_summary', {}).get('after'),
+ duration(elapsed), duration(seconds * 1000 if isinstance(seconds, (int, float)) else None),
+ ingest.get('successful_requests'), ingest.get('failed_requests'),
+ execution.get('load_workers_requested'), execution.get('max_in_flight'),
+ execution.get('db_cpus'), execution.get('db_memory'),
+ health.get('running'), health.get('oom_killed'),
+ run.get('cleanup', {}).get('verified_absent')])
+ return '\n'.join(['# Tracesbench load / lifecycle summary', '',
+ 'Total is the recorded target lifecycle; ingest is the sender wall time, '
+ 'not the entire load/readiness phase. Health is the post-load snapshot. '
+ 'Missing evidence remains blank.', '',
+ table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest',
+ 'Write success', 'Write errors', 'Load workers', 'Max in flight',
+ 'DB CPU', 'DB memory', 'Running after load', 'OOM after load',
+ 'Container removed'], rows), '',
+ render_resources([('dataset', root / 'runs' / 'generate')] +
+ [(t, root / 'runs' / t) for t in targets], table)])
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--root', type=Path, required=True)
+ parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True)
+ parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines')
+ args = parser.parse_args()
+ targets = args.targets.split(',')
+ if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets):
+ parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo')
+ report, runs = load(args.root)
+ print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml
index 86713c911ed..d265801d4a0 100644
--- a/.github/workflows/agent-observability.yml
+++ b/.github/workflows/agent-observability.yml
@@ -26,8 +26,18 @@ on:
- M
default: S
required: true
+ compare_greptimedb:
+ description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+ type: boolean
+ default: false
+ required: false
+ baseline_greptimedb_tag:
+ description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+ type: string
+ default: v1.2.1
+ required: false
greptimedb_tag:
- description: Docker Hub greptime/greptimedb tag
+ description: Candidate Docker Hub greptime/greptimedb tag
type: string
default: latest
required: true
@@ -46,9 +56,9 @@ on:
type: string
required: false
runtime_image:
- description: Aliyun runtime image (tag or digest reference)
+ description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
type: string
- default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0
+ default: auto
required: true
provider:
description: Cloud provider for the benchmark runner
@@ -89,12 +99,14 @@ on:
permissions:
contents: read
env:
+ COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+ BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
PROFILE: ${{ inputs.dataset }}
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
- RUNTIME_IMAGE: ${{ inputs.runtime_image }}
+ RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -118,6 +130,12 @@ jobs:
with:
path: greptimedb
persist-credentials: false
+ - name: Validate version comparison
+ env:
+ RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+ run: |
+ python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+ python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
- name: Resolve CI runner
id: runner
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -126,6 +144,16 @@ jobs:
python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
python3 greptimedb/tests/perf/test_ci_runner_scripts.py
python3 greptimedb/tests/perf/test_agent_observability_summary.py
+ - name: Check runtime supports shared version-comparison data
+ if: inputs.compare_greptimedb
+ run: |
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+ docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \
+ || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+ - name: Validate dataset admission in runtime
+ run: |
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+ docker run --rm "$RUNTIME_IMAGE" scripts/logbench_agent_observability_measured.py --preflight-profile "$PROFILE"
- name: Validate requested benchmark
id: inputs
shell: bash
@@ -262,11 +290,11 @@ jobs:
run: |
set -euo pipefail
actual=
- [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
+ [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
images='{}'
resolve_image() {
local target=$1 reference=$2 variable=$3 metadata resolved revision
- docker pull "$reference"
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference"
metadata=$(docker image inspect "$reference")
resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
if [[ "$target" == runtime ]]; then
@@ -289,6 +317,9 @@ jobs:
victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
esac
done
+ if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+ resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE
+ fi
jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
--arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
--arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
@@ -312,6 +343,15 @@ jobs:
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
--root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
+ - name: Load and benchmark baseline GreptimeDB
+ id: baseline_greptimedb
+ if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+ run: |-
+ GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+ --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
+ --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+ --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
+ --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
- name: Load and benchmark greptimedb
id: greptimedb
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
@@ -357,6 +397,13 @@ jobs:
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
--owner "$OWNER" --root "$ARTIFACT_ROOT"
fi
+ - name: Clean up baseline containers
+ if: always() && inputs.compare_greptimedb
+ run: |-
+ if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
+ GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
+ --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
+ fi
- name: Benchmark summary
if: always()
run: |
@@ -367,6 +414,16 @@ jobs:
run: |
python3 greptimedb/.github/scripts/agent-observability-summary.py \
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+ - name: Baseline load / lifecycle summary
+ if: always() && inputs.compare_greptimedb
+ run: |
+ echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+ python3 greptimedb/.github/scripts/agent-observability-summary.py \
+ --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+ - name: GreptimeDB version comparison
+ if: always() && inputs.compare_greptimedb
+ run: |
+ python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
- name: Upload results (no dataset)
if: always()
uses: actions/upload-artifact@v4
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d3bad06f41d..deba5010c80 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -99,6 +99,7 @@ jobs:
python3 tests/perf/test_aliyun_ecs_runner_scripts.py
python3 tests/perf/test_ci_runner_scripts.py
python3 tests/perf/test_agent_observability_summary.py
+ python3 tests/perf/test_tracesbench_summary.py
check:
if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml
index 60081d69ad9..049faaef1a7 100644
--- a/.github/workflows/tracesbench.yml
+++ b/.github/workflows/tracesbench.yml
@@ -2,6 +2,36 @@ name: Tracesbench
on:
workflow_dispatch:
inputs:
+ run_greptimedb:
+ description: Run GreptimeDB
+ type: boolean
+ default: true
+ required: false
+ run_victoriatraces:
+ description: Run VictoriaTraces
+ type: boolean
+ default: false
+ required: false
+ run_tempo:
+ description: Run Tempo
+ type: boolean
+ default: false
+ required: false
+ compare_greptimedb:
+ description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+ type: boolean
+ default: false
+ required: false
+ baseline_greptimedb_tag:
+ description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+ type: string
+ default: v1.2.1
+ required: false
+ greptimedb_tag:
+ description: Candidate Docker Hub greptime/greptimedb tag
+ type: string
+ default: latest
+ required: true
dataset:
description: Dataset profile (main is experimental; native queries may exceed memory limits)
type: choice
@@ -9,9 +39,9 @@ on:
default: sanity
required: true
runtime_image:
- description: Aliyun runtime image (tag or digest reference)
+ description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
type: string
- default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260928-4e52d0b0@sha256:39d2d87593788e0c47a54768f6323679b8e03d7dc72a8158a9b22cf29f883baa
+ default: auto
required: true
provider:
description: Cloud provider for the benchmark runner
@@ -20,7 +50,7 @@ on:
default: Aliyun
required: true
instance_type:
- description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
+ description: 'Instance type; auto = Aliyun: ecs.g9i.2xlarge, AWS: m7i.2xlarge (both 8 vCPU / 32 GiB). Enter a specific type to override.'
type: string
default: auto
required: true
@@ -45,33 +75,46 @@ on:
default: 8
required: true
db_cpus:
- description: CPU limit per DB container
+ description: CPU limit per DB container (default 8 vCPU)
type: string
default: '8'
required: true
db_memory:
- description: Memory limit per DB container
+ description: Memory limit per DB container (default 32 GiB; shared host also runs the runtime and runner)
type: string
- default: 16g
+ default: 32g
+ required: true
+ load_workers:
+ description: CPU workers for parallel import reading/parsing across all engines (separate from write concurrency)
+ type: number
+ default: 4
+ required: true
+ query_timeout_seconds:
+ description: Timeout per query request in seconds (correctness, warmup and measurement; 1..3600)
+ type: number
+ default: 300
required: true
runtime_cpus:
- description: CPU limit per runtime container
+ description: CPU limit per runtime container (0 = unrestricted)
type: string
- default: '2'
+ default: '0'
required: true
runtime_memory:
- description: Memory limit per runtime container (separate from DB memory)
+ description: Memory limit per runtime container (0 = unrestricted; separate from DB memory)
type: string
- default: 4g
+ default: '0'
required: true
permissions:
contents: read
env:
+ COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+ BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
DATASET: ${{ inputs.dataset }}
+ GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
- RUNTIME_IMAGE: ${{ inputs.runtime_image }}
+ RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -79,12 +122,18 @@ env:
JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
DB_CPUS: ${{ inputs.db_cpus }}
DB_MEMORY: ${{ inputs.db_memory }}
+ QUERY_TIMEOUT_SECONDS: ${{ inputs.query_timeout_seconds }}
+ LOAD_WORKERS: ${{ inputs.load_workers }}
jobs:
validate:
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
instance_type: ${{ steps.runner.outputs.instance_type }}
+ targets: ${{ steps.inputs.outputs.targets }}
+ greptimedb: ${{ steps.inputs.outputs.greptimedb }}
+ victoriatraces: ${{ steps.inputs.outputs.victoriatraces }}
+ tempo: ${{ steps.inputs.outputs.tempo }}
system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }}
runtime_image: ${{ steps.runtime.outputs.runtime_image }}
o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }}
@@ -93,9 +142,44 @@ jobs:
with:
path: greptimedb
persist-credentials: false
+ - name: Validate selected targets
+ id: inputs
+ env:
+ greptimedb: ${{ inputs.run_greptimedb }}
+ victoriatraces: ${{ inputs.run_victoriatraces }}
+ tempo: ${{ inputs.run_tempo }}
+ run: |
+ set -euo pipefail
+ selected=
+ for target in greptimedb victoriatraces tempo; do
+ enabled=${!target}
+ [[ "$enabled" == true || "$enabled" == false ]]
+ printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT"
+ if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi
+ done
+ [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; }
+ if [[ "$greptimedb" == true ]]; then
+ [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
+ fi
+ printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
+ - name: Validate version comparison
+ env:
+ RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+ run: |
+ python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+ python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
- name: Resolve CI runner
id: runner
- run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
+ run: |
+ set -euo pipefail
+ instance_type="$REQUESTED_INSTANCE_TYPE"
+ if [[ "$instance_type" == auto ]]; then
+ case "$PROVIDER" in
+ Aliyun) instance_type=ecs.g9i.2xlarge ;;
+ AWS) instance_type=m7i.2xlarge ;;
+ esac
+ fi
+ python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$instance_type"
- name: Validate budgets and shared runtime
id: runtime
run: |
@@ -115,19 +199,38 @@ jobs:
((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120))
[[ "$DB_CPUS" =~ ^[1-9][0-9]*$ && "$DB_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]]
- [[ "$RUNTIME_CPUS" =~ ^[1-9][0-9]*$ && "$RUNTIME_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]]
- [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
- docker pull "$RUNTIME_IMAGE"
+ [[ "$RUNTIME_CPUS" =~ ^(0|[1-9][0-9]*)$ && "$RUNTIME_MEMORY" =~ ^(0|[1-9][0-9]*[mMgG])$ ]]
+ [[ "$QUERY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,3}$ ]]
+ ((QUERY_TIMEOUT_SECONDS <= 3600))
+ [[ "$LOAD_WORKERS" =~ ^[1-9][0-9]{0,2}$ ]]
+ ((LOAD_WORKERS <= 256))
+ runtime_limits=()
+ [[ "$RUNTIME_CPUS" == 0 ]] || runtime_limits+=(--cpus "$RUNTIME_CPUS")
+ [[ "$RUNTIME_MEMORY" == 0 ]] || runtime_limits+=(--memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY")
+ [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
metadata=$(docker image inspect "$RUNTIME_IMAGE")
resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
+ if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+ docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \
+ || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+ fi
revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
[[ "$revision" =~ ^[0-9a-f]{40}$ ]]
# Refuse runtimes without the trace runner and its validated dataset/report contract.
- help=$(docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/run_tracesbench_docker.py --help)
+ help=$(docker run --rm "${runtime_limits[@]}" "$resolved" scripts/run_tracesbench_docker.py --help)
+ if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+ grep -F -- '--baseline-run' <<< "$help" \
+ || { echo 'Comparison requires a runtime with --baseline-run report support'; exit 1; }
+ fi
grep -F -- '--dataset {sanity,preflight,main}' <<< "$help"
grep -F -- '--runtime-cpus' <<< "$help"
grep -F -- '--runtime-memory' <<< "$help"
- docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/check_run_tracesbench_docker.py
+ grep -F -- '--query-timeout-seconds' <<< "$help"
+ grep -F -- '--load-workers' <<< "$help"
+ grep -F -- '--targets' <<< "$help"
+ grep -F -- '--greptimedb-image' <<< "$help"
+ docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py
printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
provision:
needs: validate
@@ -190,6 +293,7 @@ jobs:
runs-on: ${{ needs.provision.outputs.label }}
timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
env:
+ TARGETS: ${{ needs.validate.outputs.targets }}
RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }}
RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }}
RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }}
@@ -224,36 +328,83 @@ jobs:
available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
((available >= minimum_gib * 1024 * 1024)) || { echo "Need at least $minimum_gib GiB free on $path"; exit 1; }
done
- docker pull "$RESOLVED_RUNTIME_IMAGE"
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
- # Preserve the pinned catalog images and require all three targets in the report.
+ - name: Pull database images before generating data
+ run: |
+ set -euo pipefail
+ IFS=, read -ra targets <<< "$TARGETS"
+ for target in "${targets[@]}"; do
+ if [[ "$target" == greptimedb ]]; then
+ image="greptime/greptimedb:$GREPTIMEDB_TAG"
+ else
+ image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \
+ "o11ybench/workloads/tracesbench/catalogs/$target.json")
+ fi
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+ docker image inspect "$image" > "$METADATA_ROOT/$target-image.json"
+ if [[ "$target" == greptimedb ]]; then
+ resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json")
+ printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+ fi
+ done
+ if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+ image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG"
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+ docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json"
+ resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json")
+ printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+ fi
- name: Generate selected dataset once
+ id: generate
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- - name: GreptimeDB load, correctness and benchmark
+ - name: Baseline GreptimeDB load, correctness and benchmark
+ if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
run: |
- python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 \
- --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+ python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
+ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+ --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+ python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb"
+ - name: GreptimeDB load, correctness and benchmark
+ if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
+ run: |
+ python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
+ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+ python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb"
- name: VictoriaTraces load, correctness and benchmark
+ if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \
- --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Tempo load, correctness and benchmark
+ if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }}
run: |
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \
- --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- - name: Validate and render report
+ - name: GreptimeDB version comparison
+ if: always() && inputs.compare_greptimedb
run: |
- python3 o11ybench/scripts/run_tracesbench_docker.py --stage report \
+ python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
+ - name: Validate and render report
+ env:
+ COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+ run: |
+ comparison_args=()
+ if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+ comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json")
+ fi
+ python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
- name: Clean up owned containers
@@ -262,6 +413,23 @@ jobs:
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+ - name: Clean up baseline containers
+ if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
+ run: |
+ python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
+ --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+ - name: Benchmark summary
+ if: always()
+ run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
+ - name: Load / lifecycle summary
+ if: always()
+ run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+ - name: Baseline load / lifecycle summary
+ if: always() && inputs.compare_greptimedb
+ run: |
+ echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+ python3 greptimedb/.github/scripts/tracesbench-summary.py \
+ --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
- name: Stage benchmark artifacts
if: always()
run: |
@@ -269,11 +437,15 @@ jobs:
if [[ -d "$METADATA_ROOT" ]]; then
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
fi
- for entry in runs report report.json report.md; do
+ for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
if [[ -e "$RUN_ROOT/$entry" ]]; then
rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
fi
done
+ if [[ -d "$RUN_ROOT/baseline/runs" ]]; then
+ mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
+ rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
+ fi
- name: Upload HTML and results (no corpus)
if: always()
uses: actions/upload-artifact@v4
diff --git a/.github/workflows/vmbench-long-range.yml b/.github/workflows/vmbench-long-range.yml
index 357d2b21c69..aaf2dd7f437 100644
--- a/.github/workflows/vmbench-long-range.yml
+++ b/.github/workflows/vmbench-long-range.yml
@@ -20,8 +20,18 @@ on:
- 1mo
default: 1w
required: true
+ compare_greptimedb:
+ description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+ type: boolean
+ default: false
+ required: false
+ baseline_greptimedb_tag:
+ description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+ type: string
+ default: v1.2.1
+ required: false
greptimedb_tag:
- description: Docker Hub greptime/greptimedb tag
+ description: Candidate Docker Hub greptime/greptimedb tag
type: string
default: latest
required: true
@@ -31,9 +41,9 @@ on:
default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344
required: true
runtime_image:
- description: Aliyun runtime image (tag or digest reference)
+ description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
type: string
- default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29
+ default: auto
required: true
provider:
description: Cloud provider for the benchmark runner
@@ -79,11 +89,13 @@ on:
permissions:
contents: read
env:
+ COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+ BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
RANGE_PRESET: ${{ inputs.range_preset }}
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
- RUNTIME_IMAGE: ${{ inputs.runtime_image }}
+ RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }}
PROVIDER: ${{ inputs.provider }}
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -107,6 +119,12 @@ jobs:
with:
path: greptimedb
persist-credentials: false
+ - name: Validate version comparison
+ env:
+ RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+ run: |
+ python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+ python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
- name: Resolve CI runner
id: runner
run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -135,8 +153,8 @@ jobs:
((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
# Match the existing ECS workflow's provision, teardown and queue headroom.
((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120))
- [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
- docker pull "$RUNTIME_IMAGE"
+ [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
metadata=$(docker image inspect "$RUNTIME_IMAGE")
resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
@@ -146,6 +164,9 @@ jobs:
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
--run-classification controlled_server_benchmark --no-open-browser)
+ if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+ args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range")
+ fi
# Tags become target-spec fields; reject separators before passing them to the planner.
for target in greptimedb victoriametrics; do
case "$target" in
@@ -273,13 +294,13 @@ jobs:
available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
((available >= 10 * 1024 * 1024)) || { echo "Need at least 10 GiB free on $path"; exit 1; }
done
- docker pull "$RESOLVED_RUNTIME_IMAGE"
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
plan="$METADATA_ROOT/warm.json"
while IFS=$'\t' read -r target reference; do
echo "Resolving $target: $reference"
- docker pull "$reference"
+ bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference"
docker image inspect "$reference" > "$METADATA_ROOT/$target-image.json"
digest=$(jq -er '.[0].RepoDigests[0]' "$METADATA_ROOT/$target-image.json")
jq --arg target "$target" --arg image "$digest" '.targets |= map(if .name == $target then .image = $image else . end)' "$plan" > "$plan.tmp"
@@ -303,6 +324,10 @@ jobs:
if: always()
run: |
docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
+ - name: GreptimeDB version comparison
+ if: always() && inputs.compare_greptimedb
+ run: |
+ python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
- name: Stage benchmark artifacts
if: always()
run: |
diff --git a/tests/perf/test_agent_observability_summary.py b/tests/perf/test_agent_observability_summary.py
index 8223231c908..a6c1774ba0f 100644
--- a/tests/perf/test_agent_observability_summary.py
+++ b/tests/perf/test_agent_observability_summary.py
@@ -14,11 +14,13 @@
import importlib.util
import json
+import sys
import tempfile
import unittest
from pathlib import Path
SCRIPT = Path(__file__).parents[2] / '.github/scripts/agent-observability-summary.py'
+sys.path.insert(0, str(SCRIPT.parent))
spec = importlib.util.spec_from_file_location('observability_summary', SCRIPT)
summary = importlib.util.module_from_spec(spec)
spec.loader.exec_module(summary)
diff --git a/tests/perf/test_benchmark_version_comparison.py b/tests/perf/test_benchmark_version_comparison.py
new file mode 100644
index 00000000000..56c2619e81c
--- /dev/null
+++ b/tests/perf/test_benchmark_version_comparison.py
@@ -0,0 +1,138 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+SCRIPTS = Path(__file__).resolve().parents[2] / '.github/scripts'
+spec = importlib.util.spec_from_file_location('comparison', SCRIPTS / 'benchmark-version-comparison.py')
+comparison = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(comparison)
+
+
+def write(path, value):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(value))
+
+
+class VersionComparison(unittest.TestCase):
+ def test_opt_in_validation(self):
+ comparison.validate({})
+ comparison.validate({'COMPARE_GREPTIMEDB': 'false', 'GREPTIMEDB_TAG': 'invalid\n'})
+ good = dict(COMPARE_GREPTIMEDB='true', RUN_GREPTIMEDB='true',
+ BASELINE_GREPTIMEDB_TAG='v1.2.1', GREPTIMEDB_TAG='v1.3.0-beta.1')
+ comparison.validate(good)
+ for key, value in [('RUN_GREPTIMEDB', 'false'), ('BASELINE_GREPTIMEDB_TAG', ''),
+ ('BASELINE_GREPTIMEDB_TAG', 'v1.3.0-beta.1'),
+ ('BASELINE_GREPTIMEDB_TAG', 'a,image=other'), ('GREPTIMEDB_TAG', 'x\ny'),
+ ('COMPARE_GREPTIMEDB', 'yes')]:
+ with self.subTest(key=key, value=value), self.assertRaises(ValueError):
+ comparison.validate(good | {key: value})
+
+ def test_improvement_direction(self):
+ self.assertAlmostEqual(comparison.improvement(100, 80), 20)
+ self.assertAlmostEqual(comparison.improvement(100, 120), -20)
+ self.assertAlmostEqual(comparison.improvement(100, 120, True), 20)
+ for old, new in [(0, 3), (None, 4), (1, -1), (1, float('nan')), (True, 2)]:
+ self.assertIsNone(comparison.improvement(old, new))
+
+ def test_agent_result_gates(self):
+ summary = dict(passed=True, contract_sha256='contract', data_profile='S', query_window=[1, 2],
+ warmup_runs=1, measured_runs=5, per_query=[dict(query_id='Q01', result_fingerprint='answer',
+ success=5, errors=0, timeouts=0, p50_ms=100, p95_ms=100, p99_ms=100)])
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp)
+ base = root / 'baseline/greptimedb-1/greptimedb-1/measured-summary.json'
+ candidate = root / 'greptimedb-1/greptimedb-1/measured-summary.json'
+ for r in (root / 'baseline', root):
+ (r / 'greptimedb-1').mkdir(parents=True, exist_ok=True)
+ (r / 'greptimedb-1/exit-code.txt').write_text('0\n')
+ write(base, summary)
+ changed = copy.deepcopy(summary); changed['per_query'][0]['p50_ms'] = 80
+ write(candidate, changed)
+ self.assertAlmostEqual(comparison.agent(root)[0]['improvement_percent'], 20)
+ for key, value in [('errors', 1), ('timeouts', 1), ('result_fingerprint', 'wrong'), ('success', 0)]:
+ broken = copy.deepcopy(changed); broken['per_query'][0][key] = value
+ write(candidate, broken)
+ self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+ (root / 'greptimedb-1/exit-code.txt').write_text('1\n')
+ write(candidate, changed)
+ self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+ (root / 'greptimedb-1/exit-code.txt').write_text('0\n')
+ changed['query_window'] = [2, 3]; write(candidate, changed)
+ self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+ candidate.unlink()
+ self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+
+ def test_traces_uses_measured_only_and_rejects_failed_query(self):
+ answer = dict(query_id='T01', outcome='passed', fingerprint='answer', row_count=1, elapsed_ms=100)
+ sample = dict(answer, phase='measured')
+ record = dict(success=True, expected={'contracts': {'T01': {'fingerprint': 'answer', 'row_count': 1}}},
+ protocol={'runs': 5}, audit_expected={}, corpus='sanity',
+ execution=dict(db_cpus='8', db_memory='16g', load_workers_requested=4, max_in_flight=8),
+ correctness=[answer], samples=[dict(sample, phase='warmup', elapsed_ms=99999), sample, sample])
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp);base = root / 'baseline/runs/greptimedb/run.json'; candidate = root / 'runs/greptimedb/run.json'
+ write(base, record); changed = copy.deepcopy(record)
+ for s in changed['samples']:
+ if s['phase'] == 'measured': s['elapsed_ms'] = 80
+ write(candidate, changed)
+ self.assertAlmostEqual(comparison.traces(root)[0]['improvement_percent'], 20)
+ self.assertIsNone(comparison.traces(root)[1]['improvement_percent'])
+ for key, value in [('outcome', 'failed'), ('fingerprint', 'wrong'), ('row_count', 2)]:
+ broken = copy.deepcopy(changed); broken['samples'][1][key] = value; write(candidate, broken)
+ self.assertIsNone(comparison.traces(root)[0]['improvement_percent'])
+
+ def test_long_range_gates(self):
+ summary = dict(workload='vmbench-long-range', query_file_sha256='hash', start='start', end='end', step='30m',
+ profile='small', data_duration='32d', scrape_interval='30s', cache_policy_effective='warm',
+ mode='serial', connection_mode='force-close', request_timeout_seconds=120, tries=5, warmup_runs=1,
+ errors=0, timeouts=0, per_query=[dict(query_index=1, query='up', series=1, points=2,
+ success=5, errors=0, timeouts=0, p50_ms=100)])
+ with tempfile.TemporaryDirectory() as tmp:
+ root = Path(tmp); base = root/'greptimedb-baseline/results-vmbench-long-range/serial-summary.json'
+ candidate = root/'greptimedb/results-vmbench-long-range/serial-summary.json'
+ health = {phase: {target: {'state': 'Running=true ExitCode=0 OOMKilled=false'}
+ for target in ('greptimedb-baseline', 'greptimedb')}
+ for phase in ('post_load_results', 'post_query_results')}
+ health['post_load_results'] = {k: {'docker': v} for k, v in health['post_load_results'].items()}
+ write(root / 'run-manifest.json', health)
+ write(base, summary); write(candidate, summary)
+ self.assertEqual(comparison.long_range(root)[0]['improvement_percent'], 0)
+ for key, value in [('start','different'),('tries',1),('query_file_sha256','other'),('errors',1)]:
+ write(candidate, summary | {key:value})
+ self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+ write(candidate, summary)
+ broken=copy.deepcopy(health);broken['post_query_results']['greptimedb']['state']='Running=false ExitCode=137 OOMKilled=true'
+ write(root / 'run-manifest.json', broken)
+ self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+ write(root / 'run-manifest.json', health)
+ empty=copy.deepcopy(summary);empty['per_query'][0]['points']=0;write(candidate,empty)
+ self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+
+ def test_missing_results_and_rendering(self):
+ with tempfile.TemporaryDirectory() as tmp:
+ for load in (comparison.agent, comparison.traces, comparison.long_range):
+ self.assertTrue(all(r['improvement_percent'] is None for r in load(Path(tmp))))
+ rendered = comparison.render(dict(baseline_tag='old', candidate_tag='', rows=[]))
+ self.assertIn('<new|tag>', rendered)
+ self.assertIn('Results unavailable', rendered)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tests/perf/test_ci_runner_scripts.py b/tests/perf/test_ci_runner_scripts.py
index f9742079b49..c703e41c62b 100644
--- a/tests/perf/test_ci_runner_scripts.py
+++ b/tests/perf/test_ci_runner_scripts.py
@@ -16,6 +16,7 @@
"""Offline regression coverage for CI runner selection and lifecycle."""
import importlib.util
+import json
import os
import re
import subprocess
@@ -77,6 +78,67 @@ class ProviderConfigTest(unittest.TestCase):
with self.subTest(provider=provider, requested=requested):
self.assertEqual(config.resolve_instance_type(provider, requested), expected)
+ def test_traces_workflow_runner_defaults_and_overrides(self):
+ workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text()
+ step = workflow.split(" - name: Resolve CI runner\n", 1)[1].split(" - name:", 1)[0]
+ command = textwrap.dedent(step.split(" run: |\n", 1)[1])
+ with tempfile.TemporaryDirectory() as directory:
+ root = Path(directory)
+ (root / "greptimedb").symlink_to(SCRIPTS.parents[1], target_is_directory=True)
+ output = root / "output"
+ for provider, requested, expected in (
+ ("Aliyun", "auto", "ecs.g9i.2xlarge"),
+ ("AWS", "auto", "m7i.2xlarge"),
+ ("Aliyun", "ecs.c9i.4xlarge", "ecs.c9i.4xlarge"),
+ ("AWS", "c7i.4xlarge", "c7i.4xlarge"),
+ ("invalid", "auto", None),
+ ("AWS", "ecs.g9i.2xlarge", None),
+ ):
+ with self.subTest(provider=provider, requested=requested):
+ output.write_text("")
+ result = subprocess.run(
+ ["bash", "-c", command], cwd=root, capture_output=True, text=True,
+ env=dict(os.environ, PROVIDER=provider, REQUESTED_INSTANCE_TYPE=requested,
+ GITHUB_OUTPUT=str(output)), timeout=10,
+ )
+ if expected is None:
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(output.read_text(), "")
+ else:
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(output.read_text(), f"instance_type={expected}\n")
+ self.assertRegex(workflow, r"(?s)db_cpus:.*?default: '8'")
+ self.assertRegex(workflow, r"(?s)db_memory:.*?default: 32g")
+
+ def test_traces_selected_targets_and_image_tag(self):
+ workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text()
+ step = workflow.split(" - name: Validate selected targets\n", 1)[1].split(" - name:", 1)[0]
+ command = textwrap.dedent(step.split(" run: |\n", 1)[1])
+ with tempfile.TemporaryDirectory() as directory:
+ output = Path(directory) / "output"
+ for enabled, tag, expected in (
+ (("true", "false", "false"), "latest", "greptimedb"),
+ (("false", "true", "false"), "", "victoriatraces"),
+ (("false", "false", "true"), "", "tempo"),
+ (("true", "true", "true"), "v1.3.0-beta.1", "greptimedb,victoriatraces,tempo"),
+ (("false", "false", "false"), "latest", None),
+ (("true", "false", "false"), "latest; touch /tmp/unexpected", None),
+ (("true", "false", "false"), "", None),
+ (("invalid", "false", "false"), "latest", None),
+ ):
+ with self.subTest(enabled=enabled, tag=tag):
+ output.write_text("")
+ env = dict(os.environ, GREPTIMEDB_TAG=tag, GITHUB_OUTPUT=str(output))
+ env.update(zip(("greptimedb", "victoriatraces", "tempo"), enabled))
+ result = subprocess.run(["bash", "-c", command], env=env,
+ capture_output=True, text=True, timeout=5)
+ if expected is None:
+ self.assertNotEqual(result.returncode, 0)
+ self.assertNotIn("targets=", output.read_text())
+ else:
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertIn(f"targets={expected}\n", output.read_text())
+
def test_wrong_provider_and_shell_fragments_rejected(self):
for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"),
("Aliyun", "c7i.2xlarge"), ("AWS", ""),
@@ -401,6 +463,154 @@ class AwsTeardownTest(unittest.TestCase):
self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1)
self.assertEqual([c.args[1] for c in delete.call_args_list], ["i-a", "i-b"])
+class BenchmarkImagePullTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tmp.cleanup)
+ self.root = Path(self.tmp.name)
+ self.env = dict(os.environ, PATH=f"{self.root}:{os.environ['PATH']}",
+ PULL_TEST_ROOT=str(self.root))
+ self.tool("docker", '''
+ echo "$*" >> "$PULL_TEST_ROOT/docker.log"
+ count=$(wc -l < "$PULL_TEST_ROOT/docker.log")
+ codes=${PULL_TEST_CODES:-0}
+ code=$(printf '%s' "$codes" | cut -d, -f"$count")
+ exit "${code:-0}"
+ ''')
+ self.tool("timeout", '''
+ echo "$*" >> "$PULL_TEST_ROOT/timeout.log"
+ shift 2
+ if [[ ${PULL_TEST_TIMEOUT:-0} == 1 ]]; then exit 124; fi
+ exec "$@"
+ ''')
+ self.tool("sleep", 'echo "$*" >> "$PULL_TEST_ROOT/sleep.log"')
+
+ def tool(self, name, body):
+ p = self.root / name
+ p.write_text("#!/bin/bash\nset -eu\n" + textwrap.dedent(body))
+ p.chmod(0o755)
+
+ def lines(self, name):
+ p = self.root / (name + ".log")
+ return p.read_text().splitlines() if p.exists() else []
+
+ def pull(self, codes, *args):
+ self.env["PULL_TEST_CODES"] = codes
+ return subprocess.run(["bash", str(SCRIPTS / "pull-benchmark-image.sh"), *args],
+ env=self.env, capture_output=True, text=True, timeout=5)
+
+ def test_success_does_not_retry(self):
+ result = self.pull("0", "registry/test@sha256:" + "a" * 64)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(len(self.lines("docker")), 1)
+ self.assertEqual(self.lines("sleep"), [])
+ self.assertTrue(self.lines("timeout")[0].startswith("--kill-after=10s 180s docker pull "))
+
+ def test_transient_failure_then_success(self):
+ result = self.pull("1,1,0", "registry/test:tag")
+ self.assertEqual(result.returncode, 0, result.stderr)
+ self.assertEqual(self.lines("docker"), ["pull registry/test:tag"] * 3)
+ self.assertEqual(self.lines("sleep"), ["10", "20"])
+ self.assertIn("attempt 3/3", result.stderr)
+
+ def test_exhaustion_preserves_failure(self):
+ result = self.pull("1,1,7", "registry/test:tag")
+ self.assertEqual(result.returncode, 7)
+ self.assertEqual(len(self.lines("docker")), 3)
+ self.assertEqual(self.lines("sleep"), ["10", "20"])
+
+ def test_timeout_is_bounded_and_fatal(self):
+ self.env["PULL_TEST_TIMEOUT"] = "1"
+ result = self.pull("0", "registry/test:tag")
+ self.assertEqual(result.returncode, 124)
+ self.assertEqual(len(self.lines("timeout")), 3)
+ self.assertEqual(self.lines("sleep"), ["10", "20"])
+
+ def test_bad_arguments_do_not_call_docker(self):
+ for args in ([], [""], ["--help"], ["a", "b"]):
+ with self.subTest(args=args):
+ self.assertEqual(self.pull("0", *args).returncode, 2)
+ self.assertEqual(self.lines("docker"), [])
+
+ def test_traces_prefetch_executes_catalog_refs_and_stops_on_failure(self):
+ traces = (SCRIPTS.parent / 'workflows/tracesbench.yml').read_text()
+ start = traces.index(' - name: Pull database images before generating data')
+ end = traces.index(' - name: Generate selected dataset once')
+ command = textwrap.dedent(traces[start:end].split(' run: |\n', 1)[1])
+ helper = self.root / 'greptimedb/.github/scripts/pull-benchmark-image.sh'
+ helper.parent.mkdir(parents=True)
+ helper.write_text((SCRIPTS / helper.name).read_text())
+ catalogs = self.root / 'o11ybench/workloads/tracesbench/catalogs'
+ catalogs.mkdir(parents=True)
+ images = [f'registry/{name}@sha256:' + 'a' * 64
+ for name in ('greptimedb', 'victoriatraces', 'tempo')]
+ for name, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images):
+ (catalogs / f'{name}.json').write_text(json.dumps({'target_identity': {'image': image}}))
+ self.tool('docker', '''
+ echo "$*" >> "$PULL_TEST_ROOT/docker.log"
+ if [[ $1 == pull && $2 == ${PULL_FAIL_IMAGE:-} ]]; then exit 1; fi
+ if [[ $1 == image ]]; then printf '[{"RepoDigests":["greptime/greptimedb@sha256:%s"]}]\\n' "$(printf 'a%.0s' {1..64})"; fi
+ ''')
+ images[0] = 'greptime/greptimedb:v1.3.0-beta.1'
+ self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1',
+ GITHUB_ENV=str(self.root / 'env'), BASELINE_GREPTIMEDB_TAG='v1.2.1')
+ for selected, compare in ((('greptimedb',), False), (('tempo',), False),
+ (('greptimedb', 'victoriatraces', 'tempo'), False),
+ (('greptimedb',), True), (('greptimedb', 'victoriatraces', 'tempo'), True)):
+ with self.subTest(selected=selected):
+ self.env['TARGETS'] = ','.join(selected)
+ self.env['COMPARE_GREPTIMEDB'] = str(compare).lower()
+ (self.root / 'env').write_text('')
+ (self.root / 'docker.log').unlink(missing_ok=True)
+ result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
+ capture_output=True, text=True, timeout=5)
+ self.assertEqual(result.returncode, 0, result.stderr)
+ selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images)
+ if target in selected]
+ if compare:
+ selected_images.append('greptime/greptimedb:v1.2.1')
+ self.assertEqual(self.lines('docker'),
+ [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')])
+ expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
+ expected_env = expected_env if 'greptimedb' in selected else ''
+ if compare:
+ expected_env += 'BASELINE_GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
+ self.assertEqual((self.root / 'env').read_text(), expected_env)
+ (self.root / 'docker.log').unlink()
+ self.env['PULL_FAIL_IMAGE'] = images[1]
+ result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
+ capture_output=True, text=True, timeout=5)
+ self.assertNotEqual(result.returncode, 0)
+ self.assertEqual(self.lines('docker'), [f'pull {images[0]}', f'image inspect {images[0]}']
+ + [f'pull {images[1]}'] * 3)
+ self.assertNotIn(f'pull {images[2]}', self.lines('docker'))
+
+ def test_workflows_pull_before_generation(self):
+ workflows = SCRIPTS.parent / "workflows"
+ agent = (workflows / "agent-observability.yml").read_text()
+ traces = (workflows / "tracesbench.yml").read_text()
+ long_range = (workflows / "vmbench-long-range.yml").read_text()
+ for content in (agent, traces, long_range):
+ self.assertNotIn("docker pull ", content)
+ for reference in ("RUNTIME_IMAGE", "RESOLVED_RUNTIME_IMAGE", "reference"):
+ self.assertIn(f'pull-benchmark-image.sh "${reference}"', long_range)
+ self.assertLess(long_range.index('pull-benchmark-image.sh "$reference"'),
+ long_range.index('- name: Warm serial and concurrent benchmarks'))
+ self.assertIn('pull-benchmark-image.sh "$reference"', agent)
+ self.assertLess(agent.index('resolve_image runtime'), agent.index('- name: Generate dataset once'))
+ self.assertIn('pull-benchmark-image.sh "$RUNTIME_IMAGE"', traces)
+ self.assertIn('pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"', traces)
+ start = traces.index(' - name: Pull database images before generating data')
+ end = traces.index(' - name: Generate selected dataset once')
+ self.assertLess(start, end)
+ step = traces[start:end]
+ self.assertIn('IFS=, read -ra targets <<< "$TARGETS"', step)
+ self.assertIn('for target in "${targets[@]}"', step)
+ self.assertIn('o11ybench/workloads/tracesbench/catalogs/$target.json', step)
+ self.assertIn('pull-benchmark-image.sh "$image"', step)
+ self.assertIn('set -euo pipefail', step)
+ self.assertNotIn('continue-on-error', step)
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/perf/test_tracesbench_summary.py b/tests/perf/test_tracesbench_summary.py
new file mode 100644
index 00000000000..cb7c4d8e06e
--- /dev/null
+++ b/tests/perf/test_tracesbench_summary.py
@@ -0,0 +1,54 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.util
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).parents[2] / '.github/scripts/tracesbench-summary.py'
+sys.path.insert(0, str(SCRIPT.parent))
+spec = importlib.util.spec_from_file_location('tracesbench_summary', SCRIPT)
+summary = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(summary)
+
+
+class LifecycleTest(unittest.TestCase):
+ def test_ingestion_layouts(self):
+ metrics = {'wall_seconds': 0.121723417, 'successful_requests': 17, 'failed_requests': 0}
+ cases = [
+ (target, {'ingest': metrics}, ['121.7 ms', '17', '0'])
+ for target in summary.TARGETS
+ ] + [
+ ('tempo', {'transport': {'ingest': metrics}}, ['121.7 ms', '17', '0']),
+ ('tempo', {'ingest': {'wall_seconds': 0, 'successful_requests': 0, 'failed_requests': 0},
+ 'transport': {'ingest': metrics}}, ['0 µs', '0', '0']),
+ ('tempo', {'ingest': {}, 'transport': {'ingest': metrics}}, ['—', '—', '—']),
+ ('tempo', {'transport': {}}, ['—', '—', '—']),
+ ] + [(target, {}, ['—', '—', '—']) for target in summary.TARGETS] + [
+ (target, {'transport': {'ingest': metrics}}, ['—', '—', '—'])
+ for target in ('greptimedb', 'victoriatraces')
+ ]
+ with tempfile.TemporaryDirectory() as tmp:
+ for target, load, expected in cases:
+ with self.subTest(target=target, load=load):
+ text = summary.lifecycle({}, [{'target': target, 'load': load}], Path(tmp), [target])
+ row = next(line for line in text.splitlines() if line.startswith('| ' + target + ' |'))
+ cells = [cell.strip() for cell in row.split('|')[1:-1]]
+ self.assertEqual(cells[4:7], expected)
+
+
+if __name__ == '__main__':
+ unittest.main()