ci: compare baseline and candidate GreptimeDB benchmarks

Signed-off-by: WenyXu <wenymedia@gmail.com>
2026-10-18 01:42:28 +00:00 · 2026-09-30 02:08:35 +00:00
parent 81b8ed1659
commit 31646a903f
6 changed files with 463 additions and 7 deletions
@@ -0,0 +1,191 @@
+# Copyright 2023 Greptime Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Compare validated GreptimeDB query results; never gate CI on noisy latency deltas."""
+import argparse
+import html
+import json
+import math
+import os
+from pathlib import Path
+import re
+import statistics
+
+
+def read(path):
+    return json.loads(path.read_text()) if path.is_file() else {}
+
+
+def validate(env):
+    enabled = env.get('COMPARE_GREPTIMEDB', 'false')
+    if enabled not in ('true', 'false'):
+        raise ValueError('COMPARE_GREPTIMEDB must be true or false')
+    if enabled == 'false':
+        return
+    if env.get('RUN_GREPTIMEDB') != 'true':
+        raise ValueError('Version comparison requires Run GreptimeDB')
+    for key in ('BASELINE_GREPTIMEDB_TAG', 'GREPTIMEDB_TAG'):
+        if not re.fullmatch(r'[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}(@sha256:[0-9a-f]{64})?', env.get(key, '')):
+            raise ValueError(f'Invalid {key}')
+    if env['BASELINE_GREPTIMEDB_TAG'] == env['GREPTIMEDB_TAG']:
+        raise ValueError('Baseline and candidate must have different image tags')
+
+
+def same(a, b, fields):
+    return all(a.get(k) is not None and a.get(k) == b.get(k) for k in fields)
+
+
+def finite(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0
+
+
+def improvement(old, new, throughput=False):
+    if not finite(old) or not finite(new) or old <= 0:
+        return None
+    return (new / old - 1 if throughput else 1 - new / old) * 100
+
+
+def row(query, metric, old, new, compatible, reason):
+    delta = improvement(old, new, metric == 'QPS') if compatible else None
+    return dict(query=query, metric=metric, baseline=old, candidate=new,
+                improvement_percent=delta, status='comparable' if delta is not None else reason)
+
+
+def agent(root):
+    def load(r):
+        return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json')
+    a, b = load(root / 'baseline'), load(root)
+    finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)]
+    compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished)
+                  and a.get('passed') is True and b.get('passed') is True
+                  and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs')))
+    qa, qb = ({q['query_id']: q for q in x.get('per_query', [])} for x in (a, b))
+    rows = []
+    for q in sorted(qa.keys() | qb.keys()):
+        old, new = qa.get(q, {}), qb.get(q, {})
+        ok = compatible and same(old, new, ('result_fingerprint', 'success')) and old.get('success', 0) > 0
+        ok = ok and all(v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+        for key in ('p50_ms', 'p95_ms', 'p99_ms'):
+            rows.append(row(q, key, old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+    return rows
+
+
+def traces(root):
+    a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root))
+    compatible = (a.get('success') is True and b.get('success') is True
+                  and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus'))
+                  and same(a.get('execution', {}), b.get('execution', {}),
+                           ('db_cpus', 'db_memory', 'load_workers_requested', 'max_in_flight')))
+    rows = []
+    for q in (f'T{i:02}' for i in range(1, 9)):
+        values, identities, counts = [], [], []
+        ok = compatible
+        for run in (a, b):
+            correctness = [s for s in run.get('correctness', []) if s.get('query_id') == q]
+            all_samples = [s for s in run.get('samples', []) if s.get('query_id') == q]
+            measured = [s for s in all_samples if s.get('phase') == 'measured']
+            passed = bool(correctness and measured) and all(s.get('outcome') == 'passed' for s in correctness + all_samples)
+            expected = run.get('expected', {}).get('contracts', {}).get(q, {})
+            passed = passed and all(s.get('fingerprint') == expected.get('fingerprint') and s.get('fingerprint') is not None
+                                    and s.get('row_count') == expected.get('row_count') for s in correctness + all_samples)
+            elapsed = [s.get('elapsed_ms') for s in measured]
+            passed = passed and all(finite(v) for v in elapsed)
+            ok = ok and passed
+            values.append(statistics.median(elapsed) if passed else None)
+            identities.append(expected)
+            counts.append((len(correctness), len(all_samples), len(measured)))
+        ok = ok and identities[0] == identities[1] and counts[0] == counts[1]
+        rows.append(row(q, 'median_ms', *values, ok, 'failed, skipped, missing or incompatible result'))
+    return rows
+
+
+def long_range(root):
+    rows = []
+    manifest = read(root / 'run-manifest.json')
+    healthy = all(
+        all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results'
+                      else manifest.get(phase, {}).get(target, {})).get('state', '').split()
+            for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false'))
+        for phase in ('post_load_results', 'post_query_results')
+        for target in ('greptimedb-baseline', 'greptimedb'))
+    for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'):
+        a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json'))
+                for name in ('greptimedb-baseline', 'greptimedb'))
+        fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration',
+                  'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds')
+        fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds')
+        compatible = healthy and same(a, b, fields) and all(x.get('errors') == 0 and x.get('timeouts') == 0 for x in (a, b))
+        qa, qb = ({q['query_index']: q for q in x.get('per_query', [])} for x in (a, b))
+        for q in sorted(qa.keys() | qb.keys()):
+            old, new = qa.get(q, {}), qb.get(q, {})
+            ok = compatible and same(old, new, ('query', 'series', 'points'))
+            ok = ok and all(v.get('success', 0) > 0 and v.get('series', 0) > 0 and v.get('points', 0) > 0
+                            and v.get('errors') == 0 and v.get('timeouts') == 0 for v in (old, new))
+            for key in ('p50_ms', 'qps') if mode != 'serial' else ('p50_ms',):
+                rows.append(row(f'{mode}/Q{q}', 'QPS' if key == 'qps' else key,
+                                old.get(key), new.get(key), ok, 'failed, missing or incompatible result'))
+    return rows
+
+
+def render(document):
+    def cell(value):
+        return html.escape(str(value)).replace('|', '&#124;').replace('\n', ' ')
+    def value(v, metric):
+        if not finite(v):
+            return '—'
+        if metric == 'QPS':
+            return f'{v:.3f}'
+        return f'{v / 1000:.2f} s' if v >= 1000 else f'{v:.2f} ms'
+    lines = ['# GreptimeDB version comparison', '',
+             f'Baseline: `{cell(document["baseline_tag"])}` → candidate: `{cell(document["candidate_tag"])}`.', '',
+             'Same-host sequential runs with fresh databases and shared input data. Positive = improvement; '
+             'latency: (baseline − candidate) / baseline; QPS: (candidate − baseline) / baseline. '
+             'Single runs are diagnostic, not a statistically established regression. No performance threshold gates CI.', '',
+             '| Query | Metric | Baseline | Candidate | Improvement | Status |',
+             '| --- | --- | --- | --- | --- | --- |']
+    for r in document['rows']:
+        delta = r['improvement_percent']
+        lines.append('| ' + ' | '.join(map(cell, [r['query'], r['metric'], value(r['baseline'], r['metric']),
+                     value(r['candidate'], r['metric']), f'{delta:+.1f}%' if delta is not None else '—', r['status']])) + ' |')
+    if document.get('kind') == 'long-range':
+        lines += ['', 'Long-range retains the existing non-empty series/point-count checks; these do not establish equality of every returned value.']
+    if not document['rows']:
+        lines += ['', 'Results unavailable; comparison not performed.']
+    return '\n'.join(lines) + '\n'
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('command', choices=('validate', 'report'))
+    parser.add_argument('--kind', choices=('agent', 'traces', 'long-range'))
+    parser.add_argument('--root', type=Path)
+    args = parser.parse_args()
+    if args.command == 'validate':
+        validate(os.environ)
+        return
+    if args.root is None or args.kind is None:
+        parser.error('report requires --root and --kind')
+    document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''),
+                'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''),
+                'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)}
+    text = render(document)
+    # Preserve partial comparisons even if a target failed before creating output.
+    args.root.mkdir(parents=True, exist_ok=True)
+    (args.root / 'version-comparison.json').write_text(json.dumps(document, indent=2, allow_nan=False) + '\n')
+    (args.root / 'version-comparison.md').write_text(text)
+    print(text, end='')
+
+
+if __name__ == '__main__':
+    main()
@@ -26,8 +26,18 @@ on:
          - M
        default: S
        required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
      greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
        type: string
        default: latest
        required: true
@@ -89,6 +99,8 @@ on:
 permissions:
  contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
  PROFILE: ${{ inputs.dataset }}
  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
  CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
@@ -118,6 +130,12 @@ jobs:
        with:
          path: greptimedb
          persist-credentials: false
+      - name: Validate version comparison
+        env:
+          RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+        run: |
+          python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
      - name: Resolve CI runner
        id: runner
        run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -126,6 +144,12 @@ jobs:
          python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
          python3 greptimedb/tests/perf/test_ci_runner_scripts.py
          python3 greptimedb/tests/perf/test_agent_observability_summary.py
+      - name: Check runtime supports shared version-comparison data
+        if: inputs.compare_greptimedb
+        run: |
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+          docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
      - name: Validate requested benchmark
        id: inputs
        shell: bash
@@ -289,6 +313,9 @@ jobs:
              victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
            esac
          done
+          if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+            resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE
+          fi
          jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
            --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
            --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
@@ -312,6 +339,15 @@ jobs:
            --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
            --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
          cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
+      - name: Load and benchmark baseline GreptimeDB
+        id: baseline_greptimedb
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+        run: |-
+          GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
+            --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
      - name: Load and benchmark greptimedb
        id: greptimedb
        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
@@ -357,6 +393,13 @@ jobs:
            GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
              --owner "$OWNER" --root "$ARTIFACT_ROOT"
          fi
+      - name: Clean up baseline containers
+        if: always() && inputs.compare_greptimedb
+        run: |-
+          if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
+            GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
+              --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
+          fi
      - name: Benchmark summary
        if: always()
        run: |
@@ -367,6 +410,16 @@ jobs:
        run: |
          python3 greptimedb/.github/scripts/agent-observability-summary.py \
            --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: Baseline load / lifecycle summary
+        if: always() && inputs.compare_greptimedb
+        run: |
+          echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+          python3 greptimedb/.github/scripts/agent-observability-summary.py \
+            --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: GreptimeDB version comparison
+        if: always() && inputs.compare_greptimedb
+        run: |
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
      - name: Upload results (no dataset)
        if: always()
        uses: actions/upload-artifact@v4
@@ -17,8 +17,18 @@ on:
        type: boolean
        default: false
        required: false
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
      greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
        type: string
        default: latest
        required: true
@@ -97,6 +107,8 @@ on:
 permissions:
  contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
  DATASET: ${{ inputs.dataset }}
  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
  RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
@@ -150,6 +162,12 @@ jobs:
          [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
        fi
        printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
    - name: Resolve CI runner
      id: runner
      run: |
@@ -193,6 +211,10 @@ jobs:
        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
        metadata=$(docker image inspect "$RUNTIME_IMAGE")
        resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+        fi
        revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
        [[ "$revision" =~ ^[0-9a-f]{40}$ ]]
        # Refuse runtimes without the trace runner and its validated dataset/report contract.
@@ -323,12 +345,26 @@ jobs:
            printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
          fi
        done
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG"
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+          docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json"
+          resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json")
+          printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+        fi
    - name: Generate selected dataset once
      id: generate
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Baseline GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: GreptimeDB load, correctness and benchmark
      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
      run: |
@@ -361,12 +397,27 @@ jobs:
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Clean up baseline containers
+      if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
+          --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Benchmark summary
      if: always()
      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
    - name: Load / lifecycle summary
      if: always()
      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: Baseline load / lifecycle summary
+      if: always() && inputs.compare_greptimedb
+      run: |
+        echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+        python3 greptimedb/.github/scripts/tracesbench-summary.py \
+          --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
    - name: Stage benchmark artifacts
      if: always()
      run: |
@@ -374,11 +425,15 @@ jobs:
        if [[ -d "$METADATA_ROOT" ]]; then
          rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
        fi
-        for entry in runs report report.json report.md; do
+        for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
          if [[ -e "$RUN_ROOT/$entry" ]]; then
            rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
          fi
        done
+        if [[ -d "$RUN_ROOT/baseline/runs" ]]; then
+          mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
+          rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
+        fi
    - name: Upload HTML and results (no corpus)
      if: always()
      uses: actions/upload-artifact@v4
@@ -20,8 +20,18 @@ on:
        - 1mo
        default: 1w
        required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
      greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
        type: string
        default: latest
        required: true
@@ -79,6 +89,8 @@ on:
 permissions:
  contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
  BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
  RANGE_PRESET: ${{ inputs.range_preset }}
  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
@@ -107,6 +119,12 @@ jobs:
      with:
        path: greptimedb
        persist-credentials: false
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
    - name: Resolve CI runner
      id: runner
      run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -146,6 +164,9 @@ jobs:
        args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
          --run-classification controlled_server_benchmark --no-open-browser)
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range")
+        fi
        # Tags become target-spec fields; reject separators before passing them to the planner.
        for target in greptimedb victoriametrics; do
          case "$target" in
@@ -303,6 +324,10 @@ jobs:
      if: always()
      run: |
        docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
    - name: Stage benchmark artifacts
      if: always()
      run: |
@@ -0,0 +1,124 @@
+import copy
+import importlib.util
+import json
+from pathlib import Path
+import tempfile
+import unittest
+
+SCRIPTS = Path(__file__).resolve().parents[2] / '.github/scripts'
+spec = importlib.util.spec_from_file_location('comparison', SCRIPTS / 'benchmark-version-comparison.py')
+comparison = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(comparison)
+
+
+def write(path, value):
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(value))
+
+
+class VersionComparison(unittest.TestCase):
+    def test_opt_in_validation(self):
+        comparison.validate({})
+        comparison.validate({'COMPARE_GREPTIMEDB': 'false', 'GREPTIMEDB_TAG': 'invalid\n'})
+        good = dict(COMPARE_GREPTIMEDB='true', RUN_GREPTIMEDB='true',
+                    BASELINE_GREPTIMEDB_TAG='v1.2.1', GREPTIMEDB_TAG='v1.3.0-beta.1')
+        comparison.validate(good)
+        for key, value in [('RUN_GREPTIMEDB', 'false'), ('BASELINE_GREPTIMEDB_TAG', ''),
+                           ('BASELINE_GREPTIMEDB_TAG', 'v1.3.0-beta.1'),
+                           ('BASELINE_GREPTIMEDB_TAG', 'a,image=other'), ('GREPTIMEDB_TAG', 'x\ny'),
+                           ('COMPARE_GREPTIMEDB', 'yes')]:
+            with self.subTest(key=key, value=value), self.assertRaises(ValueError):
+                comparison.validate(good | {key: value})
+
+    def test_improvement_direction(self):
+        self.assertAlmostEqual(comparison.improvement(100, 80), 20)
+        self.assertAlmostEqual(comparison.improvement(100, 120), -20)
+        self.assertAlmostEqual(comparison.improvement(100, 120, True), 20)
+        for old, new in [(0, 3), (None, 4), (1, -1), (1, float('nan')), (True, 2)]:
+            self.assertIsNone(comparison.improvement(old, new))
+
+    def test_agent_result_gates(self):
+        summary = dict(passed=True, contract_sha256='contract', data_profile='S', query_window=[1, 2],
+                       warmup_runs=1, measured_runs=5, per_query=[dict(query_id='Q01', result_fingerprint='answer',
+                       success=5, errors=0, timeouts=0, p50_ms=100, p95_ms=100, p99_ms=100)])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            base = root / 'baseline/greptimedb-1/greptimedb-1/measured-summary.json'
+            candidate = root / 'greptimedb-1/greptimedb-1/measured-summary.json'
+            for r in (root / 'baseline', root):
+                (r / 'greptimedb-1').mkdir(parents=True, exist_ok=True)
+                (r / 'greptimedb-1/exit-code.txt').write_text('0\n')
+            write(base, summary)
+            changed = copy.deepcopy(summary); changed['per_query'][0]['p50_ms'] = 80
+            write(candidate, changed)
+            self.assertAlmostEqual(comparison.agent(root)[0]['improvement_percent'], 20)
+            for key, value in [('errors', 1), ('timeouts', 1), ('result_fingerprint', 'wrong'), ('success', 0)]:
+                broken = copy.deepcopy(changed); broken['per_query'][0][key] = value
+                write(candidate, broken)
+                self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+            (root / 'greptimedb-1/exit-code.txt').write_text('1\n')
+            write(candidate, changed)
+            self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+            (root / 'greptimedb-1/exit-code.txt').write_text('0\n')
+            changed['query_window'] = [2, 3]; write(candidate, changed)
+            self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+            candidate.unlink()
+            self.assertTrue(all(r['improvement_percent'] is None for r in comparison.agent(root)))
+
+    def test_traces_uses_measured_only_and_rejects_failed_query(self):
+        answer = dict(query_id='T01', outcome='passed', fingerprint='answer', row_count=1, elapsed_ms=100)
+        sample = dict(answer, phase='measured')
+        record = dict(success=True, expected={'contracts': {'T01': {'fingerprint': 'answer', 'row_count': 1}}},
+                      protocol={'runs': 5}, audit_expected={}, corpus='sanity',
+                      execution=dict(db_cpus='8', db_memory='16g', load_workers_requested=4, max_in_flight=8),
+                      correctness=[answer], samples=[dict(sample, phase='warmup', elapsed_ms=99999), sample, sample])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp);base = root / 'baseline/runs/greptimedb/run.json'; candidate = root / 'runs/greptimedb/run.json'
+            write(base, record); changed = copy.deepcopy(record)
+            for s in changed['samples']:
+                if s['phase'] == 'measured': s['elapsed_ms'] = 80
+            write(candidate, changed)
+            self.assertAlmostEqual(comparison.traces(root)[0]['improvement_percent'], 20)
+            self.assertIsNone(comparison.traces(root)[1]['improvement_percent'])
+            for key, value in [('outcome', 'failed'), ('fingerprint', 'wrong'), ('row_count', 2)]:
+                broken = copy.deepcopy(changed); broken['samples'][1][key] = value; write(candidate, broken)
+                self.assertIsNone(comparison.traces(root)[0]['improvement_percent'])
+
+    def test_long_range_gates(self):
+        summary = dict(workload='vmbench-long-range', query_file_sha256='hash', start='start', end='end', step='30m',
+                       profile='small', data_duration='32d', scrape_interval='30s', cache_policy_effective='warm',
+                       mode='serial', connection_mode='force-close', request_timeout_seconds=120, tries=5, warmup_runs=1,
+                       errors=0, timeouts=0, per_query=[dict(query_index=1, query='up', series=1, points=2,
+                       success=5, errors=0, timeouts=0, p50_ms=100)])
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp); base = root/'greptimedb-baseline/results-vmbench-long-range/serial-summary.json'
+            candidate = root/'greptimedb/results-vmbench-long-range/serial-summary.json'
+            health = {phase: {target: {'state': 'Running=true ExitCode=0 OOMKilled=false'}
+                               for target in ('greptimedb-baseline', 'greptimedb')}
+                      for phase in ('post_load_results', 'post_query_results')}
+            health['post_load_results'] = {k: {'docker': v} for k, v in health['post_load_results'].items()}
+            write(root / 'run-manifest.json', health)
+            write(base, summary); write(candidate, summary)
+            self.assertEqual(comparison.long_range(root)[0]['improvement_percent'], 0)
+            for key, value in [('start','different'),('tries',1),('query_file_sha256','other'),('errors',1)]:
+                write(candidate, summary | {key:value})
+                self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+            write(candidate, summary)
+            broken=copy.deepcopy(health);broken['post_query_results']['greptimedb']['state']='Running=false ExitCode=137 OOMKilled=true'
+            write(root / 'run-manifest.json', broken)
+            self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+            write(root / 'run-manifest.json', health)
+            empty=copy.deepcopy(summary);empty['per_query'][0]['points']=0;write(candidate,empty)
+            self.assertIsNone(comparison.long_range(root)[0]['improvement_percent'])
+
+    def test_missing_results_and_rendering(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            for load in (comparison.agent, comparison.traces, comparison.long_range):
+                self.assertTrue(all(r['improvement_percent'] is None for r in load(Path(tmp))))
+        rendered = comparison.render(dict(baseline_tag='old', candidate_tag='<new|tag>', rows=[]))
+        self.assertIn('&lt;new&#124;tag&gt;', rendered)
+        self.assertIn('Results unavailable', rendered)
+
+
+if __name__ == '__main__':
+    unittest.main()
@@ -553,10 +553,13 @@ class BenchmarkImagePullTest(unittest.TestCase):
        ''')
        images[0] = 'greptime/greptimedb:v1.3.0-beta.1'
        self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1',
-                        GITHUB_ENV=str(self.root / 'env'))
-        for selected in (('greptimedb',), ('tempo',), ('greptimedb', 'victoriatraces', 'tempo')):
+                        GITHUB_ENV=str(self.root / 'env'), BASELINE_GREPTIMEDB_TAG='v1.2.1')
+        for selected, compare in ((('greptimedb',), False), (('tempo',), False),
+                                  (('greptimedb', 'victoriatraces', 'tempo'), False),
+                                  (('greptimedb',), True), (('greptimedb', 'victoriatraces', 'tempo'), True)):
            with self.subTest(selected=selected):
                self.env['TARGETS'] = ','.join(selected)
+                self.env['COMPARE_GREPTIMEDB'] = str(compare).lower()
                (self.root / 'env').write_text('')
                (self.root / 'docker.log').unlink(missing_ok=True)
                result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
@@ -564,10 +567,15 @@ class BenchmarkImagePullTest(unittest.TestCase):
                self.assertEqual(result.returncode, 0, result.stderr)
                selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images)
                                   if target in selected]
+                if compare:
+                    selected_images.append('greptime/greptimedb:v1.2.1')
                self.assertEqual(self.lines('docker'),
                                 [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')])
                expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
-                self.assertEqual((self.root / 'env').read_text(), expected_env if 'greptimedb' in selected else '')
+                expected_env = expected_env if 'greptimedb' in selected else ''
+                if compare:
+                    expected_env += 'BASELINE_GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
+                self.assertEqual((self.root / 'env').read_text(), expected_env)
        (self.root / 'docker.log').unlink()
        self.env['PULL_FAIL_IMAGE'] = images[1]
        result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,