ci: enhance benchmark workflows with version comparisons and diagnostics (#9383)

* ci: update Agent and Traces benchmark runtime Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: use unrestricted benchmark runtimes by default Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: retry benchmark image pulls before generating datasets Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: increase tracesbench defaults to 8 CPU and 32 GiB Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: configure tracesbench query timeout and update runtime Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: update tracesbench runtime and configure import scan workers Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: use shared parallel tracesbench import preparation Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: update tracesbench runtime to preserve Tempo trace requests Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: summarize tracesbench query results and lifecycle Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: show benchmark resource usage and update parallel loader runtime Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(ci): resolve shared summary module in existing test harness Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: select tracesbench engines and GreptimeDB image version Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: update benchmark runtime with parallel Agent preparation Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: select benchmark runtime registry by cloud provider Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: retry long-range benchmark image pulls Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: compare baseline and candidate GreptimeDB benchmarks Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: pin published dual-version benchmark runtime Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: prepare agent benchmark indexes with updated runtime Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: validate agent admission and render traces version comparison Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: use runtime with traces index preparation Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: use consolidated runtime for benchmark version reports Signed-off-by: WenyXu <wenymedia@gmail.com> * test: add license header to benchmark comparison tests Signed-off-by: WenyXu <wenymedia@gmail.com> * fix(ci): read legacy Tempo ingestion metrics in traces summaries Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: require tracesbench query plan artifacts Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: use optimized tracesbench report runtime Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: update tracesbench runtime to merged report validation Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com> (cherry picked from commit 6e1a137411) Signed-off-by: WenyXu <wenymedia@gmail.com>
2026-10-12 23:12:28 +00:00 · 2026-10-12 12:49:21 +00:00
parent 3796c478e2
commit f26e128b8a
13 changed files with 1187 additions and 41 deletions
@@ -18,6 +18,8 @@ import json
 import math
 from pathlib import Path

+from benchmark_resource_summary import render_resources
+
 TARGETS = ('greptimedb', 'clickhouse', 'victorialogs')


@@ -201,6 +203,7 @@ def lifecycle(root, targets):
            rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else
                         'invalid timestamp units (legacy artifact)', entry.get('exit_code')])
    lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), '']
+    lines.append(render_resources([('dataset', root)] + [(t, root / f'{t}-1') for t in targets], table))
    return '\n'.join(lines)


@@ -0,0 +1,191 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas."""
+import argparse
+import html
+import json
+import math
+import os
+from pathlib import Path
+import re
+import statistics
+
+
+def read(path):
+    return json.loads(path.read_text()) if path.is_file() else {}
+
+
+def validate(env):
+    enabled = env.get('COMPARE_GREPTIMEDB', 'false')
+    if enabled not in ('true', 'false'):
+        raise ValueError('COMPARE_GREPTIMEDB must be true or false')
+    if enabled == 'false':
+        return
+    if env.get('RUN_GREPTIMEDB') != 'true':
+        raise ValueError('Version comparison requires Run GreptimeDB')
+    for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'):
+        if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')):
+            raise ValueError(f'Invalid {key}')
+    if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']:
+        raise ValueError('Baseline and candidate must have different image tags')
+
+
+def same(a, b, fields):
+    return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields)
+
+
+def finite(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def improvement(old, new, throughput=False):
+    if not finite(old) or not finite(new) or old <= 0:
+        return None
+    return (new / old - 1 if throughput else 1 - new / old) * 100
+
+
+def row(query, metric, old, new, compatible, reason):
+    delta = improvement(old, new, metric == 'QPS') if compatible else None
+    return dict(query=query, metric=metric, baseline=old, candidate=new,
+                improvement_percent=delta, status='comparable' if delta is not None else reason)
+
+
+def agent(root):
+    def load(r):
+        return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json')
+    a, b = load(root / 'baseline'), load(root)
+    finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)]
+    compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished)
+                  and a.get('passed') is True and b.get('passed') is True
+                  and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs')))
+    qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b))
+    rows = []
+    for q in sorted(qa.keys() | qb.keys()):
+        old, new = qa.get(q, {}), qb.get(q, {})
+        ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0
+        ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+        for key in ('p50_ms', 'p95_ms', 'p99_ms'):
+            rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+    return rows
+
+
+def traces(root):
+    a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root))
+    compatible = (a.get('success') is True and b.get('success') is True
+                  and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus'))
+                  and same(a.get('execution', {}), b.get('execution', {}),
+                           ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight')))
+    rows = []
+    for q in (f'T{i:02}' for i in range(1, 9)):
+        values, identities, counts = [], [], []
+        ok = compatible
+        for run in (a, b):
+            correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q]
+            all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q]
+            measured = [s for s in all_samples if s.get('phase') == 'measured']
+            passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples)
+            expected = run.get('expected', {}).get('contracts', {}).get(q, {})
+            passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None
+                                    and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples)
+            elapsed = [s.get('elapsed_ms') for s in measured]
+            passed = passed and all(finite(v) for v in elapsed)
+            ok = ok and passed
+            values.append(statistics.median(elapsed) if passed else None)
+            identities.append(expected)
+            counts.append((len(correctness), len(all_samples), len(measured)))
+        ok = ok and identities[0] == identities[1] and counts[0] == counts[1]
+        rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result'))
+    return rows
+
+
+def long_range(root):
+    rows = []
+    manifest = read(root / 'run-manifest.json')
+    healthy = all(
+        all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results'
+                      else manifest.get(phase, {}).get(target, {})).get('state', '').split()
+            for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false'))
+        for phase in ('post_load_results', 'post_query_results')
+        for target in ('greptimedb-baseline', 'greptimedb'))
+    for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'):
+        a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json'))
+                for name in ('greptimedb-baseline', 'greptimedb'))
+        fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration',
+                  'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds')
+        fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds')
+        compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b))
+        qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b))
+        for q in sorted(qa.keys() | qb.keys()):
+            old, new = qa.get(q, {}), qb.get(q, {})
+            ok = compatible and same(old, new, ('query', 'series', 'points'))
+            ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0
+                            and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+            for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',):
+                rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key,
+                                old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+    return rows
+
+
+def render(document):
+    def cell(value):
+        return html.escape(str(value)).replace('|', '&#124;').replace('\n', ' ')
+    def value(v, metric):
+        if not finite(v):
+            return '—'
+        if metric == 'QPS':
+            return f'{v:.3f}'
+        return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms'
+    lines = ['# GreptimeDB version comparison', '',
+             f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '',
+             'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; '
+             'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. '
+             'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '',
+             '| Query | Metric | Baseline | Candidate | Improvement | Status |',
+             '| --- | --- | --- | --- | --- | --- |']
+    for r in document['rows']:
+        delta = r['improvement_percent']
+        lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']),
+                     value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |')
+    if document.get('kind') == 'long-range':
+        lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.']
+    if not document['rows']:
+        lines += ['', 'Results unavailable; comparison not performed.']
+    return '\n'.join(lines) + '\n'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('command', choices=('validate', 'report'))
+    parser.add_argument('--kind', choices=('agent', 'traces', 'long-range'))
+    parser.add_argument('--root', type=Path)
+    args = parser.parse_args()
+    if args.command == 'validate':
+        validate(os.environ)
+        return
+    if args.root is None or args.kind is None:
+        parser.error('report requires --root and --kind')
+    document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''),
+                'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''),
+                'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)}
+    text = render(document)
+    # Preserve partial comparisons even if a target failed before creating output.
+    args.root.mkdir(parents=True, exist_ok=True)
+    (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n')
+    (args.root / 'version-comparison.md').write_text(text)
+    print(text, end='')
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,99 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Render shared O11yBench sampled resource artifacts for lifecycle summaries."""
+import json
+import math
+
+
+def number(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def metric(values, key, statistic):
+    entry = values.get(key, {})
+    value = entry.get(statistic)
+    return value if number(entry.get('samples')) and entry['samples'] > 0 and number(value) else None
+
+
+def decimal(value, suffix=''):
+    return f'{value:.2f}{suffix}' if number(value) else '—'
+
+
+def memory(value):
+    if not number(value):
+        return '—'
+    for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'):
+        if value < 1024 or unit == 'TiB':
+            return decimal(value, f' {unit}')
+        value /= 1024
+
+
+def capacities(path):
+    # Quotas are metadata, not recomputed CPU statistics. Bound reads on long runs.
+    result = {}
+    try:
+        with path.open() as source:
+            for index, line in enumerate(source):
+                if index == 256:
+                    break
+                record = json.loads(line)
+                cores = record.get('host', {}).get('cpu_count')
+                if number(cores) and cores > 0:
+                    result.setdefault('host', cores)
+                for role, values in record.get('containers', {}).items():
+                    quota = values.get('configured_cpu_limit_cores')
+                    if number(quota):
+                        result.setdefault(role, quota)
+                if all(role in result for role in ('host', 'runtime', 'db')):
+                    break
+    except (OSError, ValueError, TypeError, AttributeError):
+        pass
+    return result
+
+
+def render_resources(scopes, table):
+    rows = []
+    for scope, directory in scopes:
+        try:
+            summary = json.loads((directory / 'resources.summary.json').read_text())
+            phases = summary.get('phases', {}) if summary.get('schema') == 'o11ybench-resources-v1' else {}
+        except (OSError, ValueError, TypeError, AttributeError):
+            phases = {}
+        if not isinstance(phases, dict) or not phases:
+            rows.append([scope, 'unavailable', *['—'] * 9])
+            continue
+        limits = capacities(directory / 'resources.jsonl')
+        for phase, data in phases.items():
+            for role in ('host', 'runtime', 'db'):
+                values = data.get('host', {}) if role == 'host' else data.get('containers', {}).get(role, {})
+                if role == 'db' and scope == 'dataset':
+                    continue
+                quota = limits.get(role)
+                cpu_samples = values.get('cpu_used_cores', {}).get('samples')
+                rows.append([scope, phase, role, cpu_samples,
+                             decimal(metric(values, 'cpu_used_cores', 'mean')),
+                             decimal(metric(values, 'cpu_used_cores', 'max')),
+                             'unlimited' if quota == 0 else decimal(quota),
+                             memory(metric(values, 'memory_used_bytes' if role == 'host' else 'memory_usage_bytes', 'max')),
+                             decimal(metric(values, 'io_wait_percent', 'mean'), '%'),
+                             decimal(metric(values, 'io_wait_percent', 'max'), '%'),
+                             memory(metric(values, 'swap_used_bytes', 'max'))])
+    return '\n'.join(['## Sampled resources', '',
+                      'CPU is measured in cores. Mean/max summarize available samples, not continuous peaks; '
+                      'short phases may have no CPU sample. CPU capacity is host cores or the configured container quota. '
+                      'Host memory is total minus available; Docker memory excludes cache where supported. '
+                      'Swap is host-wide usage, not swap activity. Missing evidence stays —.', '',
+                      table(['Scope', 'Phase', 'Component', 'CPU samples', 'Mean CPU cores', 'Max CPU cores',
+                             'CPU capacity', 'Max memory', 'Mean I/O wait', 'Max I/O wait', 'Max swap'], rows), ''])
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Pull only: never retry database writes, benchmarks, or cloud provisioning.
+set -euo pipefail
+
+if [[ $# != 1 || -z "$1" || "$1" == -* ]]; then
+    echo "Usage: $0 IMAGE" >&2
+    exit 2
+fi
+image=$1
+attempts=3
+for ((attempt=1; attempt<=attempts; attempt++)); do
+    echo "Pulling $image (attempt $attempt/$attempts)" >&2
+    if timeout --kill-after=10s 180s docker pull "$image"; then
+        exit 0
+    else
+        status=$?
+    fi
+    if ((attempt == attempts)); then
+        echo "Failed to pull $image after $attempts attempts (exit $status)" >&2
+        exit "$status"
+    fi
+    delay=$((attempt * 10))
+    echo "Pull failed (exit $status); retrying in ${delay}s" >&2
+    sleep "$delay"
+done
@@ -0,0 +1,169 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import json
+import html
+import math
+from pathlib import Path
+
+from benchmark_resource_summary import render_resources
+
+TARGETS = ('greptimedb', 'victoriatraces', 'tempo')
+
+
+def duration(ms):
+    if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0:
+        return '—'
+    if ms < 1:
+        return f'{ms * 1000:.0f} µs'
+    if ms < 1000:
+        return f'{ms:.1f} ms'
+    if ms < 60_000:
+        return f'{ms / 1000:.2f} s'
+    minutes, seconds = divmod(round(ms / 1000), 60)
+    hours, minutes = divmod(minutes, 60)
+    return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s'
+
+
+def cell(value):
+    return html.escape(str(value if value is not None else '—')).replace('|', '&#124;').replace('\n', ' ')
+
+
+def read(path):
+    if not path.exists():
+        return {}
+    return json.loads(path.read_text())
+
+
+def table(headers, rows):
+    return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |',
+                      '| ' + ' | '.join('---' for _ in headers) + ' |'] +
+                     ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows])
+
+
+def delta(value, baseline):
+    if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)):
+        return '—'
+    return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—'
+
+
+
+def load(root):
+    report = read(root / 'report.json')
+    runs = [read(p) for p in sorted((root / 'runs').glob('*/run.json'))]
+    return report, runs
+
+
+def benchmark(report, runs, targets=TARGETS):
+    lines = ['# Tracesbench benchmark summary', '',
+             'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. '
+             'Min/median/max come from the validated report, not recalculated samples. '
+             'Counts exclude warmup. Missing or excluded results have no latency comparison.',
+             '', f'Classification: {cell(report.get("classification"))}. '
+             'See the uploaded HTML for query semantics and client-side compensation.', '']
+    statistics = {}
+    for corpus in report.get('per_corpus', []):
+        for stat in corpus.get('statistics', []):
+            if stat['target'] not in targets:
+                continue
+            statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat
+    rows, details = [], []
+    for target in targets:
+        selected = [run for run in runs if run.get('target') == target]
+        if not selected:
+            lines.append(f'- **{target}**: unavailable / not run.')
+        for run in selected:
+            lines.append(f'- **{target}**: lifecycle '
+                         f'{"completed" if run.get("success") is True else "failed / incomplete"}; '
+                         f'query failures: {len(run.get("query_failures", []))}.')
+            for qid in (f'T{i:02}' for i in range(1, 9)):
+                correct = [q for q in run.get('correctness', []) if q.get('query_id') == qid]
+                measured = [q for q in run.get('samples', [])
+                            if q.get('query_id') == qid and q.get('phase') == 'measured']
+                errors = [q for q in correct + run.get('samples', [])
+                          if q.get('query_id') == qid and q.get('outcome') not in ('passed', 'not_executed')]
+                reason = (run.get('exclusions', {}).get(qid) or
+                          run.get('execution', {}).get('query_skips', {}).get(qid))
+                stat = statistics.get((run.get('corpus'), run.get('round'), target, qid), {})
+                base = statistics.get((run.get('corpus'), run.get('round'), 'greptimedb', qid), {})
+                status = ('failed' if errors else 'excluded' if reason else
+                          'passed' if stat else
+                          'correctness passed; report unavailable' if correct and
+                          all(q.get('outcome') == 'passed' for q in correct) else 'unavailable / incomplete')
+                rows.append([run.get('corpus'), run.get('round'), qid, target, status,
+                             duration(stat.get('min_ms')), duration(stat.get('median_ms')),
+                             duration(stat.get('max_ms')),
+                             delta(stat.get('median_ms'), base.get('median_ms')) if target != 'greptimedb' else '—',
+                             sum(q.get('outcome') == 'passed' for q in measured),
+                             sum(q.get('outcome') != 'passed' for q in measured)])
+                if reason or errors:
+                    messages = [str(q.get('error') or q.get('outcome')) for q in errors]
+                    details.append(f'- **{target} / {qid}**: {cell(str(reason or "") + " " + "; ".join(messages))[:4000]}')
+    lines += ['', table(['Dataset', 'Round', 'Query', 'DB', 'Status', 'Min', 'Median', 'Max',
+                         'Median Δ', 'Measured success', 'Measured errors'], rows), '',
+              '<details>', '<summary>Query failures and exclusions</summary>', '', *details, '', '</details>']
+    if not report:
+        lines += ['', 'Validated report unavailable; raw outcomes are shown without latency statistics.']
+    return '\n'.join(lines)
+
+
+def lifecycle(report, runs, root, targets=TARGETS):
+    rows = []
+    for target in targets:
+        selected = [r for r in runs if r.get('target') == target] or [{}]
+        for run in selected:
+            load = run.get('load', {})
+            ingest = load.get('ingest', {})
+            if target == 'tempo' and 'ingest' not in load:
+                ingest = load.get('transport', {}).get('ingest', {})
+            health = run.get('post_load_health', {})
+            execution = run.get('execution', {})
+            start, finish = run.get('started_unix_ns'), run.get('finished_unix_ns')
+            elapsed = (finish - start) / 1_000_000 if isinstance(start, int) and isinstance(finish, int) else None
+            seconds = ingest.get('wall_seconds')
+            rows.append([target, run.get('corpus'), run.get('load_summary', {}).get('after'),
+                         duration(elapsed), duration(seconds * 1000 if isinstance(seconds, (int, float)) else None),
+                         ingest.get('successful_requests'), ingest.get('failed_requests'),
+                         execution.get('load_workers_requested'), execution.get('max_in_flight'),
+                         execution.get('db_cpus'), execution.get('db_memory'),
+                         health.get('running'), health.get('oom_killed'),
+                         run.get('cleanup', {}).get('verified_absent')])
+    return '\n'.join(['# Tracesbench load / lifecycle summary', '',
+                       'Total is the recorded target lifecycle; ingest is the sender wall time, '
+                       'not the entire load/readiness phase. Health is the post-load snapshot. '
+                       'Missing evidence remains blank.', '',
+                       table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest',
+                              'Write success', 'Write errors', 'Load workers', 'Max in flight',
+                              'DB CPU', 'DB memory', 'Running after load', 'OOM after load',
+                              'Container removed'], rows), '',
+                       render_resources([('dataset', root / 'runs' / 'generate')] +
+                                        [(t, root / 'runs' / t) for t in targets], table)])
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--root', type=Path, required=True)
+    parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True)
+    parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines')
+    args = parser.parse_args()
+    targets = args.targets.split(',')
+    if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets):
+        parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo')
+    report, runs = load(args.root)
+    print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets))
+
+
+if __name__ == '__main__':
+    main()
@@ -26,8 +26,18 @@ on:
          - M
        default: S
        required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
      greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
        type: string
        default: latest
        required: true
@@ -46,9 +56,9 @@ on:
        type: string
        required: false
      runtime_image:
-        description: Aliyun runtime image (tag or digest reference)
+        description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
        type: string
-        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0
+        default: auto
        required: true
      provider:
        description: Cloud provider for the benchmark runner
@@ -89,12 +99,14 @@ on:
 permissions:
  contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
  PROFILE: ${{ inputs.dataset }}
  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
  CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
  VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
  O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
-  RUNTIME_IMAGE: ${{ inputs.runtime_image }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }}
  PROVIDER: ${{ inputs.provider }}
  REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
  SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -118,6 +130,12 @@ jobs:
        with:
          path: greptimedb
          persist-credentials: false
+      - name: Validate version comparison
+        env:
+          RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+        run: |
+          python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
      - name: Resolve CI runner
        id: runner
        run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -126,6 +144,16 @@ jobs:
          python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
          python3 greptimedb/tests/perf/test_ci_runner_scripts.py
          python3 greptimedb/tests/perf/test_agent_observability_summary.py
+      - name: Check runtime supports shared version-comparison data
+        if: inputs.compare_greptimedb
+        run: |
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+          docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+      - name: Validate dataset admission in runtime
+        run: |
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+          docker run --rm "$RUNTIME_IMAGE" scripts/logbench_agent_observability_measured.py --preflight-profile "$PROFILE"
      - name: Validate requested benchmark
        id: inputs
        shell: bash
@@ -262,11 +290,11 @@ jobs:
        run: |
          set -euo pipefail
          actual=
-          [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
+          [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
          images='{}'
          resolve_image() {
            local target=$1 reference=$2 variable=$3 metadata resolved revision
-            docker pull "$reference"
+            bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference"
            metadata=$(docker image inspect "$reference")
            resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
            if [[ "$target" == runtime ]]; then
@@ -289,6 +317,9 @@ jobs:
              victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
            esac
          done
+          if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+            resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE
+          fi
          jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
            --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
            --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
@@ -312,6 +343,15 @@ jobs:
            --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
            --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
          cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
+      - name: Load and benchmark baseline GreptimeDB
+        id: baseline_greptimedb
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+        run: |-
+          GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
+            --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
      - name: Load and benchmark greptimedb
        id: greptimedb
        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
@@ -357,6 +397,13 @@ jobs:
            GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
              --owner "$OWNER" --root "$ARTIFACT_ROOT"
          fi
+      - name: Clean up baseline containers
+        if: always() && inputs.compare_greptimedb
+        run: |-
+          if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
+            GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
+              --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
+          fi
      - name: Benchmark summary
        if: always()
        run: |
@@ -367,6 +414,16 @@ jobs:
        run: |
          python3 greptimedb/.github/scripts/agent-observability-summary.py \
            --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: Baseline load / lifecycle summary
+        if: always() && inputs.compare_greptimedb
+        run: |
+          echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+          python3 greptimedb/.github/scripts/agent-observability-summary.py \
+            --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: GreptimeDB version comparison
+        if: always() && inputs.compare_greptimedb
+        run: |
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
      - name: Upload results (no dataset)
        if: always()
        uses: actions/upload-artifact@v4
@@ -99,6 +99,7 @@ jobs:
          python3 tests/perf/test_aliyun_ecs_runner_scripts.py
          python3 tests/perf/test_ci_runner_scripts.py
          python3 tests/perf/test_agent_observability_summary.py
+          python3 tests/perf/test_tracesbench_summary.py

  check:
    if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name != 'pull_request' || github.event.pull_request.draft == false || github.run_attempt > 1) }}
@@ -2,6 +2,36 @@ name: Tracesbench
 on:
  workflow_dispatch:
    inputs:
+      run_greptimedb:
+        description: Run GreptimeDB
+        type: boolean
+        default: true
+        required: false
+      run_victoriatraces:
+        description: Run VictoriaTraces
+        type: boolean
+        default: false
+        required: false
+      run_tempo:
+        description: Run Tempo
+        type: boolean
+        default: false
+        required: false
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
+      greptimedb_tag:
+        description: Candidate Docker Hub greptime/greptimedb tag
+        type: string
+        default: latest
+        required: true
      dataset:
        description: Dataset profile (main is experimental; native queries may exceed memory limits)
        type: choice
@@ -9,9 +39,9 @@ on:
        default: sanity
        required: true
      runtime_image:
-        description: Aliyun runtime image (tag or digest reference)
+        description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
        type: string
-        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260928-4e52d0b0@sha256:39d2d87593788e0c47a54768f6323679b8e03d7dc72a8158a9b22cf29f883baa
+        default: auto
        required: true
      provider:
        description: Cloud provider for the benchmark runner
@@ -20,7 +50,7 @@ on:
        default: Aliyun
        required: true
      instance_type:
-        description: 'Instance type; auto = Aliyun: ecs.c9i.2xlarge, AWS: c7i.2xlarge (both 8 vCPU / 16 GiB). Enter a specific type to override.'
+        description: 'Instance type; auto = Aliyun: ecs.g9i.2xlarge, AWS: m7i.2xlarge (both 8 vCPU / 32 GiB). Enter a specific type to override.'
        type: string
        default: auto
        required: true
@@ -45,33 +75,46 @@ on:
        default: 8
        required: true
      db_cpus:
-        description: CPU limit per DB container
+        description: CPU limit per DB container (default 8 vCPU)
        type: string
        default: '8'
        required: true
      db_memory:
-        description: Memory limit per DB container
+        description: Memory limit per DB container (default 32 GiB; shared host also runs the runtime and runner)
        type: string
-        default: 16g
+        default: 32g
+        required: true
+      load_workers:
+        description: CPU workers for parallel import reading/parsing across all engines (separate from write concurrency)
+        type: number
+        default: 4
+        required: true
+      query_timeout_seconds:
+        description: Timeout per query request in seconds (correctness, warmup and measurement; 1..3600)
+        type: number
+        default: 300
        required: true
      runtime_cpus:
-        description: CPU limit per runtime container
+        description: CPU limit per runtime container (0 = unrestricted)
        type: string
-        default: '2'
+        default: '0'
        required: true
      runtime_memory:
-        description: Memory limit per runtime container (separate from DB memory)
+        description: Memory limit per runtime container (0 = unrestricted; separate from DB memory)
        type: string
-        default: 4g
+        default: '0'
        required: true
 permissions:
  contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
  DATASET: ${{ inputs.dataset }}
+  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
  RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
  RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
  BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
-  RUNTIME_IMAGE: ${{ inputs.runtime_image }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }}
  PROVIDER: ${{ inputs.provider }}
  REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
  SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -79,12 +122,18 @@ env:
  JANITOR_TTL_HOURS: ${{ inputs.janitor_ttl_hours }}
  DB_CPUS: ${{ inputs.db_cpus }}
  DB_MEMORY: ${{ inputs.db_memory }}
+  QUERY_TIMEOUT_SECONDS: ${{ inputs.query_timeout_seconds }}
+  LOAD_WORKERS: ${{ inputs.load_workers }}
 jobs:
  validate:
    runs-on: ubuntu-latest
    timeout-minutes: 15
    outputs:
      instance_type: ${{ steps.runner.outputs.instance_type }}
+      targets: ${{ steps.inputs.outputs.targets }}
+      greptimedb: ${{ steps.inputs.outputs.greptimedb }}
+      victoriatraces: ${{ steps.inputs.outputs.victoriatraces }}
+      tempo: ${{ steps.inputs.outputs.tempo }}
      system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }}
      runtime_image: ${{ steps.runtime.outputs.runtime_image }}
      o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }}
@@ -93,9 +142,44 @@ jobs:
      with:
        path: greptimedb
        persist-credentials: false
+    - name: Validate selected targets
+      id: inputs
+      env:
+        greptimedb: ${{ inputs.run_greptimedb }}
+        victoriatraces: ${{ inputs.run_victoriatraces }}
+        tempo: ${{ inputs.run_tempo }}
+      run: |
+        set -euo pipefail
+        selected=
+        for target in greptimedb victoriatraces tempo; do
+          enabled=${!target}
+          [[ "$enabled" == true || "$enabled" == false ]]
+          printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT"
+          if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi
+        done
+        [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; }
+        if [[ "$greptimedb" == true ]]; then
+          [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
+        fi
+        printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
    - name: Resolve CI runner
      id: runner
-      run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
+      run: |
+        set -euo pipefail
+        instance_type="$REQUESTED_INSTANCE_TYPE"
+        if [[ "$instance_type" == auto ]]; then
+          case "$PROVIDER" in
+            Aliyun) instance_type=ecs.g9i.2xlarge ;;
+            AWS) instance_type=m7i.2xlarge ;;
+          esac
+        fi
+        python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$instance_type"
    - name: Validate budgets and shared runtime
      id: runtime
      run: |
@@ -115,19 +199,38 @@ jobs:
        ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
        ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120))
        [[ "$DB_CPUS" =~ ^[1-9][0-9]*$ && "$DB_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]]
-        [[ "$RUNTIME_CPUS" =~ ^[1-9][0-9]*$ && "$RUNTIME_MEMORY" =~ ^[1-9][0-9]*[mMgG]$ ]]
-        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
-        docker pull "$RUNTIME_IMAGE"
+        [[ "$RUNTIME_CPUS" =~ ^(0|[1-9][0-9]*)$ && "$RUNTIME_MEMORY" =~ ^(0|[1-9][0-9]*[mMgG])$ ]]
+        [[ "$QUERY_TIMEOUT_SECONDS" =~ ^[1-9][0-9]{0,3}$ ]]
+        ((QUERY_TIMEOUT_SECONDS <= 3600))
+        [[ "$LOAD_WORKERS" =~ ^[1-9][0-9]{0,2}$ ]]
+        ((LOAD_WORKERS <= 256))
+        runtime_limits=()
+        [[ "$RUNTIME_CPUS" == 0 ]] || runtime_limits+=(--cpus "$RUNTIME_CPUS")
+        [[ "$RUNTIME_MEMORY" == 0 ]] || runtime_limits+=(--memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY")
+        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
        metadata=$(docker image inspect "$RUNTIME_IMAGE")
        resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+        fi
        revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
        [[ "$revision" =~ ^[0-9a-f]{40}$ ]]
        # Refuse runtimes without the trace runner and its validated dataset/report contract.
-        help=$(docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/run_tracesbench_docker.py --help)
+        help=$(docker run --rm "${runtime_limits[@]}" "$resolved" scripts/run_tracesbench_docker.py --help)
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          grep -F -- '--baseline-run' <<< "$help" \
+            || { echo 'Comparison requires a runtime with --baseline-run report support'; exit 1; }
+        fi
        grep -F -- '--dataset {sanity,preflight,main}' <<< "$help"
        grep -F -- '--runtime-cpus' <<< "$help"
        grep -F -- '--runtime-memory' <<< "$help"
-        docker run --rm --cpus "$RUNTIME_CPUS" --memory "$RUNTIME_MEMORY" --memory-swap "$RUNTIME_MEMORY" "$resolved" scripts/check_run_tracesbench_docker.py
+        grep -F -- '--query-timeout-seconds' <<< "$help"
+        grep -F -- '--load-workers' <<< "$help"
+        grep -F -- '--targets' <<< "$help"
+        grep -F -- '--greptimedb-image' <<< "$help"
+        docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py
        printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
  provision:
    needs: validate
@@ -190,6 +293,7 @@ jobs:
    runs-on: ${{ needs.provision.outputs.label }}
    timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
    env:
+      TARGETS: ${{ needs.validate.outputs.targets }}
      RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }}
      RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }}
      RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }}
@@ -224,36 +328,83 @@ jobs:
          available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
          ((available >= minimum_gib * 1024 * 1024)) || { echo "Need at least $minimum_gib GiB free on $path"; exit 1; }
        done
-        docker pull "$RESOLVED_RUNTIME_IMAGE"
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
        docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
        test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
-    # Preserve the pinned catalog images and require all three targets in the report.
+    - name: Pull database images before generating data
+      run: |
+        set -euo pipefail
+        IFS=, read -ra targets <<< "$TARGETS"
+        for target in "${targets[@]}"; do
+          if [[ "$target" == greptimedb ]]; then
+            image="greptime/greptimedb:$GREPTIMEDB_TAG"
+          else
+            image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \
+              "o11ybench/workloads/tracesbench/catalogs/$target.json")
+          fi
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+          docker image inspect "$image" > "$METADATA_ROOT/$target-image.json"
+          if [[ "$target" == greptimedb ]]; then
+            resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json")
+            printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+          fi
+        done
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG"
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+          docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json"
+          resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json")
+          printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+        fi
    - name: Generate selected dataset once
+      id: generate
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
-    - name: GreptimeDB load, correctness and benchmark
+    - name: Baseline GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
      run: |
-        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 \
-          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+        python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb"
+    - name: GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+        python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb"
    - name: VictoriaTraces load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \
-          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Tempo load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }}
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \
-          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
-    - name: Validate and render report
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
      run: |
-        python3 o11ybench/scripts/run_tracesbench_docker.py --stage report \
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
+    - name: Validate and render report
+      env:
+        COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+      run: |
+        comparison_args=()
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json")
+        fi
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Clean up owned containers
@@ -262,6 +413,23 @@ jobs:
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Clean up baseline containers
+      if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
+          --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Benchmark summary
+      if: always()
+      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
+    - name: Load / lifecycle summary
+      if: always()
+      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: Baseline load / lifecycle summary
+      if: always() && inputs.compare_greptimedb
+      run: |
+        echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+        python3 greptimedb/.github/scripts/tracesbench-summary.py \
+          --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
    - name: Stage benchmark artifacts
      if: always()
      run: |
@@ -269,11 +437,15 @@ jobs:
        if [[ -d "$METADATA_ROOT" ]]; then
          rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
        fi
-        for entry in runs report report.json report.md; do
+        for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
          if [[ -e "$RUN_ROOT/$entry" ]]; then
            rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
          fi
        done
+        if [[ -d "$RUN_ROOT/baseline/runs" ]]; then
+          mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
+          rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
+        fi
    - name: Upload HTML and results (no corpus)
      if: always()
      uses: actions/upload-artifact@v4
@@ -20,8 +20,18 @@ on:
        - 1mo
        default: 1w
        required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
      greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
        type: string
        default: latest
        required: true
@@ -31,9 +41,9 @@ on:
        default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344
        required: true
      runtime_image:
-        description: Aliyun runtime image (tag or digest reference)
+        description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
        type: string
-        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260918-545dc570@sha256:0ff03e7e690575e875bb85b3054a03486dd9359073546dc715da177153cf7a29
+        default: auto
        required: true
      provider:
        description: Cloud provider for the benchmark runner
@@ -79,11 +89,13 @@ on:
 permissions:
  contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
  BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
  RANGE_PRESET: ${{ inputs.range_preset }}
  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
  VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
-  RUNTIME_IMAGE: ${{ inputs.runtime_image }}
+  RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }}
  PROVIDER: ${{ inputs.provider }}
  REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
  SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
@@ -107,6 +119,12 @@ jobs:
      with:
        path: greptimedb
        persist-credentials: false
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
    - name: Resolve CI runner
      id: runner
      run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -135,8 +153,8 @@ jobs:
        ((BENCHMARK_TIMEOUT_MINUTES <= 360 && JANITOR_TTL_HOURS <= 168))
        # Match the existing ECS workflow's provision, teardown and queue headroom.
        ((JANITOR_TTL_HOURS * 60 >= BENCHMARK_TIMEOUT_MINUTES + 120))
-        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* ]]
-        docker pull "$RUNTIME_IMAGE"
+        [[ "$RUNTIME_IMAGE" == *.cr.aliyuncs.com/* || "$RUNTIME_IMAGE" == public.ecr.aws/* ]]
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
        metadata=$(docker image inspect "$RUNTIME_IMAGE")
        resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
        revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
@@ -146,6 +164,9 @@ jobs:
        args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
          --run-classification controlled_server_benchmark --no-open-browser)
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range")
+        fi
        # Tags become target-spec fields; reject separators before passing them to the planner.
        for target in greptimedb victoriametrics; do
          case "$target" in
@@ -273,13 +294,13 @@ jobs:
          available=$(df -Pk "$path" | awk 'NR == 2 {print $4}')
          ((available >= 10 * 1024 * 1024)) || { echo "Need at least 10 GiB free on $path"; exit 1; }
        done
-        docker pull "$RESOLVED_RUNTIME_IMAGE"
+        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
        docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
        test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
        plan="$METADATA_ROOT/warm.json"
        while IFS=$'\t' read -r target reference; do
          echo "Resolving $target: $reference"
-          docker pull "$reference"
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$reference"
          docker image inspect "$reference" > "$METADATA_ROOT/$target-image.json"
          digest=$(jq -er '.[0].RepoDigests[0]' "$METADATA_ROOT/$target-image.json")
          jq --arg target "$target" --arg image "$digest" '.targets |= map(if .name == $target then .image = $image else . end)' "$plan" > "$plan.tmp"
@@ -303,6 +324,10 @@ jobs:
      if: always()
      run: |
        docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
    - name: Stage benchmark artifacts
      if: always()
      run: |
@@ -14,11 +14,13 @@

 import importlib.util
 import json
+import sys
 import tempfile
 import unittest
 from pathlib import Path

 SCRIPT = Path(__file__).parents[2] / '.github/scripts/agent-observability-summary.py'
+sys.path.insert(0, str(SCRIPT.parent))
 spec = importlib.util.spec_from_file_location('observability_summary', SCRIPT)
 summary = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(summary)
@@ -0,0 +1,138 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+SCRIPTS = Path(__file__).resolve().parents[2] / '.github/scripts'
+spec = importlib.util.spec_from_file_location('comparison', SCRIPTS / 'benchmark-version-comparison.py')
+comparison = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(comparison)
+
+
+def write(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value))
+
+
+class VersionComparison(unittest.TestCase):
+    def test_opt_in_validation(self):
+        comparison.validate({})
+        comparison.validate({'COMPARE_GREPTIMEDB': 'false', 'GREPTIMEDB_TAG': 'invalid\n'})
+        good = dict(COMPARE_GREPTIMEDB='true', RUN_GREPTIMEDB='true',
+                    BASELINE_GREPTIMEDB_TAG='v1.2.1', GREPTIMEDB_TAG='v1.3.0-beta.1')
+        comparison.validate(good)
+        for key, value in [('RUN_GREPTIMEDB', 'false'), ('BASELINE_GREPTIMEDB_TAG', ''),
+                           ('BASELINE_GREPTIMEDB_TAG', 'v1.3.0-beta.1'),
+                           ('BASELINE_GREPTIMEDB_TAG', 'a,image=other'), ('GREPTIMEDB_TAG', 'x\ny'),
+                           ('COMPARE_GREPTIMEDB', 'yes')]:
+            with self.subTest(key=key, value=value), self.assertRaises(ValueError):
+                comparison.validate(good | {key: value})
+
+    def test_improvement_direction(self):
+        self.assertAlmostEqual(comparison.improvement(100, 80), 20)
+        self.assertAlmostEqual(comparison.improvement(100, 120), -20)
+        self.assertAlmostEqual(comparison.improvement(100, 120, True), 20)
+        for old, new in [(0, 3), (None, 4), (1, -1), (1, float('nan')), (True, 2)]:
+            self.assertIsNone(comparison.improvement(old, new))
+
+    def test_agent_result_gates(self):
+        summary = dict(passed=True, contract_sha256='contract', data_profile='S', query_window=[1, 2],
+                       warmup_runs=1, measured_runs=5, per_query=[dict(query_id='Q01', result_fingerprint='answer',
+                       success=5, errors=0, timeouts=0, p50_ms=100, p95_ms=100, p99_ms=100)])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            base = root / 'baseline/greptimedb-1/greptimedb-1/measured-summary.json'
+            candidate = root / 'greptimedb-1/greptimedb-1/measured-summary.json'
+            for r in (root / 'baseline', root):
+                (r / 'greptimedb-1').mkdir(parents=True, exist_ok=True)
+                (r / 'greptimedb-1/exit-code.txt').write_text('0\n')
+            write(base, summary)
+            changed = copy.deepcopy(summary); changed['per_query'][0]['p50_ms'] = 80
+            write(candidate, changed)
+            self.assertAlmostEqual(comparison.agent(root)[0]['improvement_percent'], 20)
+            for key, value in [('errors', 1), ('timeouts', 1), ('result_fingerprint', 'wrong'), ('success', 0)]:
+                broken = copy.deepcopy(changed); broken['per_query'][0][key] = value
+                write(candidate, broken)
+                self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+            (root / 'greptimedb-1/exit-code.txt').write_text('1\n')
+            write(candidate, changed)
+            self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+            (root / 'greptimedb-1/exit-code.txt').write_text('0\n')
+            changed['query_window'] = [2, 3]; write(candidate, changed)
+            self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+            candidate.unlink()
+            self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+
+    def test_traces_uses_measured_only_and_rejects_failed_query(self):
+        answer = dict(query_id='T01', outcome='passed', fingerprint='answer', row_count=1, elapsed_ms=100)
+        sample = dict(answer, phase='measured')
+        record = dict(success=True, expected={'contracts': {'T01': {'fingerprint': 'answer', 'row_count': 1}}},
+                      protocol={'runs': 5}, audit_expected={}, corpus='sanity',
+                      execution=dict(db_cpus='8', db_memory='16g', load_workers_requested=4, max_in_flight=8),
+                      correctness=[answer], samples=[dict(sample, phase='warmup', elapsed_ms=99999), sample, sample])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp);base = root / 'baseline/runs/greptimedb/run.json'; candidate = root / 'runs/greptimedb/run.json'
+            write(base, record); changed = copy.deepcopy(record)
+            for s in changed['samples']:
+                if s['phase'] == 'measured': s['elapsed_ms'] = 80
+            write(candidate, changed)
+            self.assertAlmostEqual(comparison.traces(root)[0]['improvement_percent'], 20)
+            self.assertIsNone(comparison.traces(root)[1]['improvement_percent'])
+            for key, value in [('outcome', 'failed'), ('fingerprint', 'wrong'), ('row_count', 2)]:
+                broken = copy.deepcopy(changed); broken['samples'][1][key] = value; write(candidate, broken)
+                self.assertIsNone(comparison.traces(root)[0]['improvement_percent'])
+
+    def test_long_range_gates(self):
+        summary = dict(workload='vmbench-long-range', query_file_sha256='hash', start='start', end='end', step='30m',
+                       profile='small', data_duration='32d', scrape_interval='30s', cache_policy_effective='warm',
+                       mode='serial', connection_mode='force-close', request_timeout_seconds=120, tries=5, warmup_runs=1,
+                       errors=0, timeouts=0, per_query=[dict(query_index=1, query='up', series=1, points=2,
+                       success=5, errors=0, timeouts=0, p50_ms=100)])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); base = root/'greptimedb-baseline/results-vmbench-long-range/serial-summary.json'
+            candidate = root/'greptimedb/results-vmbench-long-range/serial-summary.json'
+            health = {phase: {target: {'state': 'Running=true ExitCode=0 OOMKilled=false'}
+                               for target in ('greptimedb-baseline', 'greptimedb')}
+                      for phase in ('post_load_results', 'post_query_results')}
+            health['post_load_results'] = {k: {'docker': v} for k, v in health['post_load_results'].items()}
+            write(root / 'run-manifest.json', health)
+            write(base, summary); write(candidate, summary)
+            self.assertEqual(comparison.long_range(root)[0]['improvement_percent'], 0)
+            for key, value in [('start','different'),('tries',1),('query_file_sha256','other'),('errors',1)]:
+                write(candidate, summary | {key:value})
+                self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+            write(candidate, summary)
+            broken=copy.deepcopy(health);broken['post_query_results']['greptimedb']['state']='Running=false ExitCode=137 OOMKilled=true'
+            write(root / 'run-manifest.json', broken)
+            self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+            write(root / 'run-manifest.json', health)
+            empty=copy.deepcopy(summary);empty['per_query'][0]['points']=0;write(candidate,empty)
+            self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+
+    def test_missing_results_and_rendering(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            for load in (comparison.agent, comparison.traces, comparison.long_range):
+                self.assertTrue(all(r['improvement_percent'] is None for r in load(Path(tmp))))
+        rendered = comparison.render(dict(baseline_tag='old', candidate_tag='<new|tag>', rows=[]))
+        self.assertIn('&lt;new&#124;tag&gt;', rendered)
+        self.assertIn('Results unavailable', rendered)
+
+
+if __name__ == '__main__':
+    unittest.main()
@@ -16,6 +16,7 @@
 """Offline regression coverage for CI runner selection and lifecycle."""

 import importlib.util
+import json
 import os
 import re
 import subprocess
@@ -77,6 +78,67 @@ class ProviderConfigTest(unittest.TestCase):
            with self.subTest(provider=provider, requested=requested):
                self.assertEqual(config.resolve_instance_type(provider, requested), expected)

+    def test_traces_workflow_runner_defaults_and_overrides(self):
+        workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text()
+        step = workflow.split("    - name: Resolve CI runner\n", 1)[1].split("    - name:", 1)[0]
+        command = textwrap.dedent(step.split("      run: |\n", 1)[1])
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "greptimedb").symlink_to(SCRIPTS.parents[1], target_is_directory=True)
+            output = root / "output"
+            for provider, requested, expected in (
+                ("Aliyun", "auto", "ecs.g9i.2xlarge"),
+                ("AWS", "auto", "m7i.2xlarge"),
+                ("Aliyun", "ecs.c9i.4xlarge", "ecs.c9i.4xlarge"),
+                ("AWS", "c7i.4xlarge", "c7i.4xlarge"),
+                ("invalid", "auto", None),
+                ("AWS", "ecs.g9i.2xlarge", None),
+            ):
+                with self.subTest(provider=provider, requested=requested):
+                    output.write_text("")
+                    result = subprocess.run(
+                        ["bash", "-c", command], cwd=root, capture_output=True, text=True,
+                        env=dict(os.environ, PROVIDER=provider, REQUESTED_INSTANCE_TYPE=requested,
+                                 GITHUB_OUTPUT=str(output)), timeout=10,
+                    )
+                    if expected is None:
+                        self.assertNotEqual(result.returncode, 0)
+                        self.assertEqual(output.read_text(), "")
+                    else:
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertEqual(output.read_text(), f"instance_type={expected}\n")
+        self.assertRegex(workflow, r"(?s)db_cpus:.*?default: '8'")
+        self.assertRegex(workflow, r"(?s)db_memory:.*?default: 32g")
+
+    def test_traces_selected_targets_and_image_tag(self):
+        workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text()
+        step = workflow.split("    - name: Validate selected targets\n", 1)[1].split("    - name:", 1)[0]
+        command = textwrap.dedent(step.split("      run: |\n", 1)[1])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory) / "output"
+            for enabled, tag, expected in (
+                (("true", "false", "false"), "latest", "greptimedb"),
+                (("false", "true", "false"), "", "victoriatraces"),
+                (("false", "false", "true"), "", "tempo"),
+                (("true", "true", "true"), "v1.3.0-beta.1", "greptimedb,victoriatraces,tempo"),
+                (("false", "false", "false"), "latest", None),
+                (("true", "false", "false"), "latest; touch /tmp/unexpected", None),
+                (("true", "false", "false"), "", None),
+                (("invalid", "false", "false"), "latest", None),
+            ):
+                with self.subTest(enabled=enabled, tag=tag):
+                    output.write_text("")
+                    env = dict(os.environ, GREPTIMEDB_TAG=tag, GITHUB_OUTPUT=str(output))
+                    env.update(zip(("greptimedb", "victoriatraces", "tempo"), enabled))
+                    result = subprocess.run(["bash", "-c", command], env=env,
+                                            capture_output=True, text=True, timeout=5)
+                    if expected is None:
+                        self.assertNotEqual(result.returncode, 0)
+                        self.assertNotIn("targets=", output.read_text())
+                    else:
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertIn(f"targets={expected}\n", output.read_text())
+
    def test_wrong_provider_and_shell_fragments_rejected(self):
        for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"),
                                    ("Aliyun", "c7i.2xlarge"), ("AWS", ""),
@@ -401,6 +463,154 @@ class AwsTeardownTest(unittest.TestCase):
            self.assertEqual(teardown.teardown(Mock(), "owner/repo", "token", run_id="123-2"), 1)
            self.assertEqual([c.args[1] for c in delete.call_args_list], ["i-a", "i-b"])

+class BenchmarkImagePullTest(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.env = dict(os.environ, PATH=f"{self.root}:{os.environ['PATH']}",
+                        PULL_TEST_ROOT=str(self.root))
+        self.tool("docker", '''
+            echo "$*" >> "$PULL_TEST_ROOT/docker.log"
+            count=$(wc -l < "$PULL_TEST_ROOT/docker.log")
+            codes=${PULL_TEST_CODES:-0}
+            code=$(printf '%s' "$codes" | cut -d, -f"$count")
+            exit "${code:-0}"
+        ''')
+        self.tool("timeout", '''
+            echo "$*" >> "$PULL_TEST_ROOT/timeout.log"
+            shift 2
+            if [[ ${PULL_TEST_TIMEOUT:-0} == 1 ]]; then exit 124; fi
+            exec "$@"
+        ''')
+        self.tool("sleep", 'echo "$*" >> "$PULL_TEST_ROOT/sleep.log"')
+
+    def tool(self, name, body):
+        p = self.root / name
+        p.write_text("#!/bin/bash\nset -eu\n" + textwrap.dedent(body))
+        p.chmod(0o755)
+
+    def lines(self, name):
+        p = self.root / (name + ".log")
+        return p.read_text().splitlines() if p.exists() else []
+
+    def pull(self, codes, *args):
+        self.env["PULL_TEST_CODES"] = codes
+        return subprocess.run(["bash", str(SCRIPTS / "pull-benchmark-image.sh"), *args],
+                              env=self.env, capture_output=True, text=True, timeout=5)
+
+    def test_success_does_not_retry(self):
+        result = self.pull("0", "registry/test@sha256:" + "a" * 64)
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(len(self.lines("docker")), 1)
+        self.assertEqual(self.lines("sleep"), [])
+        self.assertTrue(self.lines("timeout")[0].startswith("--kill-after=10s 180s docker pull "))
+
+    def test_transient_failure_then_success(self):
+        result = self.pull("1,1,0", "registry/test:tag")
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.lines("docker"), ["pull registry/test:tag"] * 3)
+        self.assertEqual(self.lines("sleep"), ["10", "20"])
+        self.assertIn("attempt 3/3", result.stderr)
+
+    def test_exhaustion_preserves_failure(self):
+        result = self.pull("1,1,7", "registry/test:tag")
+        self.assertEqual(result.returncode, 7)
+        self.assertEqual(len(self.lines("docker")), 3)
+        self.assertEqual(self.lines("sleep"), ["10", "20"])
+
+    def test_timeout_is_bounded_and_fatal(self):
+        self.env["PULL_TEST_TIMEOUT"] = "1"
+        result = self.pull("0", "registry/test:tag")
+        self.assertEqual(result.returncode, 124)
+        self.assertEqual(len(self.lines("timeout")), 3)
+        self.assertEqual(self.lines("sleep"), ["10", "20"])
+
+    def test_bad_arguments_do_not_call_docker(self):
+        for args in ([], [""], ["--help"], ["a", "b"]):
+            with self.subTest(args=args):
+                self.assertEqual(self.pull("0", *args).returncode, 2)
+        self.assertEqual(self.lines("docker"), [])
+
+    def test_traces_prefetch_executes_catalog_refs_and_stops_on_failure(self):
+        traces = (SCRIPTS.parent / 'workflows/tracesbench.yml').read_text()
+        start = traces.index('    - name: Pull database images before generating data')
+        end = traces.index('    - name: Generate selected dataset once')
+        command = textwrap.dedent(traces[start:end].split('      run: |\n', 1)[1])
+        helper = self.root / 'greptimedb/.github/scripts/pull-benchmark-image.sh'
+        helper.parent.mkdir(parents=True)
+        helper.write_text((SCRIPTS / helper.name).read_text())
+        catalogs = self.root / 'o11ybench/workloads/tracesbench/catalogs'
+        catalogs.mkdir(parents=True)
+        images = [f'registry/{name}@sha256:' + 'a' * 64
+                  for name in ('greptimedb', 'victoriatraces', 'tempo')]
+        for name, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images):
+            (catalogs / f'{name}.json').write_text(json.dumps({'target_identity': {'image': image}}))
+        self.tool('docker', '''
+            echo "$*" >> "$PULL_TEST_ROOT/docker.log"
+            if [[ $1 == pull && $2 == ${PULL_FAIL_IMAGE:-} ]]; then exit 1; fi
+            if [[ $1 == image ]]; then printf '[{"RepoDigests":["greptime/greptimedb@sha256:%s"]}]\\n' "$(printf 'a%.0s' {1..64})"; fi
+        ''')
+        images[0] = 'greptime/greptimedb:v1.3.0-beta.1'
+        self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1',
+                        GITHUB_ENV=str(self.root / 'env'), BASELINE_GREPTIMEDB_TAG='v1.2.1')
+        for selected, compare in ((('greptimedb',), False), (('tempo',), False),
+                                  (('greptimedb', 'victoriatraces', 'tempo'), False),
+                                  (('greptimedb',), True), (('greptimedb', 'victoriatraces', 'tempo'), True)):
+            with self.subTest(selected=selected):
+                self.env['TARGETS'] = ','.join(selected)
+                self.env['COMPARE_GREPTIMEDB'] = str(compare).lower()
+                (self.root / 'env').write_text('')
+                (self.root / 'docker.log').unlink(missing_ok=True)
+                result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
+                                        capture_output=True, text=True, timeout=5)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images)
+                                   if target in selected]
+                if compare:
+                    selected_images.append('greptime/greptimedb:v1.2.1')
+                self.assertEqual(self.lines('docker'),
+                                 [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')])
+                expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
+                expected_env = expected_env if 'greptimedb' in selected else ''
+                if compare:
+                    expected_env += 'BASELINE_GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
+                self.assertEqual((self.root / 'env').read_text(), expected_env)
+        (self.root / 'docker.log').unlink()
+        self.env['PULL_FAIL_IMAGE'] = images[1]
+        result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
+                                capture_output=True, text=True, timeout=5)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(self.lines('docker'), [f'pull {images[0]}', f'image inspect {images[0]}']
+                         + [f'pull {images[1]}'] * 3)
+        self.assertNotIn(f'pull {images[2]}', self.lines('docker'))
+
+    def test_workflows_pull_before_generation(self):
+        workflows = SCRIPTS.parent / "workflows"
+        agent = (workflows / "agent-observability.yml").read_text()
+        traces = (workflows / "tracesbench.yml").read_text()
+        long_range = (workflows / "vmbench-long-range.yml").read_text()
+        for content in (agent, traces, long_range):
+            self.assertNotIn("docker pull ", content)
+        for reference in ("RUNTIME_IMAGE", "RESOLVED_RUNTIME_IMAGE", "reference"):
+            self.assertIn(f'pull-benchmark-image.sh "${reference}"', long_range)
+        self.assertLess(long_range.index('pull-benchmark-image.sh "$reference"'),
+                        long_range.index('- name: Warm serial and concurrent benchmarks'))
+        self.assertIn('pull-benchmark-image.sh "$reference"', agent)
+        self.assertLess(agent.index('resolve_image runtime'), agent.index('- name: Generate dataset once'))
+        self.assertIn('pull-benchmark-image.sh "$RUNTIME_IMAGE"', traces)
+        self.assertIn('pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"', traces)
+        start = traces.index('    - name: Pull database images before generating data')
+        end = traces.index('    - name: Generate selected dataset once')
+        self.assertLess(start, end)
+        step = traces[start:end]
+        self.assertIn('IFS=, read -ra targets <<< "$TARGETS"', step)
+        self.assertIn('for target in "${targets[@]}"', step)
+        self.assertIn('o11ybench/workloads/tracesbench/catalogs/$target.json', step)
+        self.assertIn('pull-benchmark-image.sh "$image"', step)
+        self.assertIn('set -euo pipefail', step)
+        self.assertNotIn('continue-on-error', step)
+

 if __name__ == "__main__":
    unittest.main()
@@ -0,0 +1,54 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.util
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+SCRIPT = Path(__file__).parents[2] / '.github/scripts/tracesbench-summary.py'
+sys.path.insert(0, str(SCRIPT.parent))
+spec = importlib.util.spec_from_file_location('tracesbench_summary', SCRIPT)
+summary = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(summary)
+
+
+class LifecycleTest(unittest.TestCase):
+    def test_ingestion_layouts(self):
+        metrics = {'wall_seconds': 0.121723417, 'successful_requests': 17, 'failed_requests': 0}
+        cases = [
+            (target, {'ingest': metrics}, ['121.7 ms', '17', '0'])
+            for target in summary.TARGETS
+        ] + [
+            ('tempo', {'transport': {'ingest': metrics}}, ['121.7 ms', '17', '0']),
+            ('tempo', {'ingest': {'wall_seconds': 0, 'successful_requests': 0, 'failed_requests': 0},
+                       'transport': {'ingest': metrics}}, ['0 µs', '0', '0']),
+            ('tempo', {'ingest': {}, 'transport': {'ingest': metrics}}, ['—', '—', '—']),
+            ('tempo', {'transport': {}}, ['—', '—', '—']),
+        ] + [(target, {}, ['—', '—', '—']) for target in summary.TARGETS] + [
+            (target, {'transport': {'ingest': metrics}}, ['—', '—', '—'])
+            for target in ('greptimedb', 'victoriatraces')
+        ]
+        with tempfile.TemporaryDirectory() as tmp:
+            for target, load, expected in cases:
+                with self.subTest(target=target, load=load):
+                    text = summary.lifecycle({}, [{'target': target, 'load': load}], Path(tmp), [target])
+                    row = next(line for line in text.splitlines() if line.startswith('| ' + target + ' |'))
+                    cells = [cell.strip() for cell in row.split('|')[1:-1]]
+                    self.assertEqual(cells[4:7], expected)
+
+
+if __name__ == '__main__':
+    unittest.main()