# Copyright 2023 Greptime Team # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import json import html import math from pathlib import Path from benchmark_resource_summary import render_resources TARGETS = ('greptimedb', 'victoriatraces', 'tempo') def duration(ms): if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0: return '—' if ms < 1: return f'{ms * 1000:.0f} µs' if ms < 1000: return f'{ms:.1f} ms' if ms < 60_000: return f'{ms / 1000:.2f} s' minutes, seconds = divmod(round(ms / 1000), 60) hours, minutes = divmod(minutes, 60) return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s' def cell(value): return html.escape(str(value if value is not None else '—')).replace('|', '|').replace('\n', ' ') def read(path): if not path.exists(): return {} return json.loads(path.read_text()) def table(headers, rows): return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |', '| ' + ' | '.join('---' for _ in headers) + ' |'] + ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows]) def delta(value, baseline): if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)): return '—' return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—' def load(root): report = read(root / 'report.json') runs = [read(p) for p in sorted((root / 'runs').glob('*/run.json'))] return report, runs def benchmark(report, runs, targets=TARGETS): lines = ['# Tracesbench benchmark summary', '', 'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. ' 'Min/median/max come from the validated report, not recalculated samples. ' 'Counts exclude warmup. Missing or excluded results have no latency comparison.', '', f'Classification: {cell(report.get("classification"))}. ' 'See the uploaded HTML for query semantics and client-side compensation.', ''] statistics = {} for corpus in report.get('per_corpus', []): for stat in corpus.get('statistics', []): if stat['target'] not in targets: continue statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat rows, details = [], [] for target in targets: selected = [run for run in runs if run.get('target') == target] if not selected: lines.append(f'- **{target}**: unavailable / not run.') for run in selected: lines.append(f'- **{target}**: lifecycle ' f'{"completed" if run.get("success") is True else "failed / incomplete"}; ' f'query failures: {len(run.get("query_failures", []))}.') for qid in (f'T{i:02}' for i in range(1, 9)): correct = [q for q in run.get('correctness', []) if q.get('query_id') == qid] measured = [q for q in run.get('samples', []) if q.get('query_id') == qid and q.get('phase') == 'measured'] errors = [q for q in correct + run.get('samples', []) if q.get('query_id') == qid and q.get('outcome') not in ('passed', 'not_executed')] reason = (run.get('exclusions', {}).get(qid) or run.get('execution', {}).get('query_skips', {}).get(qid)) stat = statistics.get((run.get('corpus'), run.get('round'), target, qid), {}) base = statistics.get((run.get('corpus'), run.get('round'), 'greptimedb', qid), {}) status = ('failed' if errors else 'excluded' if reason else 'passed' if stat else 'correctness passed; report unavailable' if correct and all(q.get('outcome') == 'passed' for q in correct) else 'unavailable / incomplete') rows.append([run.get('corpus'), run.get('round'), qid, target, status, duration(stat.get('min_ms')), duration(stat.get('median_ms')), duration(stat.get('max_ms')), delta(stat.get('median_ms'), base.get('median_ms')) if target != 'greptimedb' else '—', sum(q.get('outcome') == 'passed' for q in measured), sum(q.get('outcome') != 'passed' for q in measured)]) if reason or errors: messages = [str(q.get('error') or q.get('outcome')) for q in errors] details.append(f'- **{target} / {qid}**: {cell(str(reason or "") + " " + "; ".join(messages))[:4000]}') lines += ['', table(['Dataset', 'Round', 'Query', 'DB', 'Status', 'Min', 'Median', 'Max', 'Median Δ', 'Measured success', 'Measured errors'], rows), '', '
', 'Query failures and exclusions', '', *details, '', '
'] if not report: lines += ['', 'Validated report unavailable; raw outcomes are shown without latency statistics.'] return '\n'.join(lines) def lifecycle(report, runs, root, targets=TARGETS): rows = [] for target in targets: selected = [r for r in runs if r.get('target') == target] or [{}] for run in selected: load = run.get('load', {}) ingest = load.get('ingest', {}) if target == 'tempo' and 'ingest' not in load: ingest = load.get('transport', {}).get('ingest', {}) health = run.get('post_load_health', {}) execution = run.get('execution', {}) start, finish = run.get('started_unix_ns'), run.get('finished_unix_ns') elapsed = (finish - start) / 1_000_000 if isinstance(start, int) and isinstance(finish, int) else None seconds = ingest.get('wall_seconds') rows.append([target, run.get('corpus'), run.get('load_summary', {}).get('after'), duration(elapsed), duration(seconds * 1000 if isinstance(seconds, (int, float)) else None), ingest.get('successful_requests'), ingest.get('failed_requests'), execution.get('load_workers_requested'), execution.get('max_in_flight'), execution.get('db_cpus'), execution.get('db_memory'), health.get('running'), health.get('oom_killed'), run.get('cleanup', {}).get('verified_absent')]) return '\n'.join(['# Tracesbench load / lifecycle summary', '', 'Total is the recorded target lifecycle; ingest is the sender wall time, ' 'not the entire load/readiness phase. Health is the post-load snapshot. ' 'Missing evidence remains blank.', '', table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest', 'Write success', 'Write errors', 'Load workers', 'Max in flight', 'DB CPU', 'DB memory', 'Running after load', 'OOM after load', 'Container removed'], rows), '', render_resources([('dataset', root / 'runs' / 'generate')] + [(t, root / 'runs' / t) for t in targets], table)]) def main(): parser = argparse.ArgumentParser() parser.add_argument('--root', type=Path, required=True) parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True) parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines') args = parser.parse_args() targets = args.targets.split(',') if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets): parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo') report, runs = load(args.root) print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets)) if __name__ == '__main__': main()