ci: compare baseline and candidate GreptimeDB benchmarks

Signed-off-by: WenyXu <wenymedia@gmail.com>

.github/workflows/agent-observability.yml

@@ -26,8 +26,18 @@ on:
           - M
         default: S
         required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
       greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
         type: string
         default: latest
         required: true
@@ -89,6 +99,8 @@ on:
 permissions:
   contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
   PROFILE: ${{ inputs.dataset }}
   GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
   CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
@@ -118,6 +130,12 @@ jobs:
         with:
           path: greptimedb
           persist-credentials: false
+      - name: Validate version comparison
+        env:
+          RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+        run: |
+          python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
       - name: Resolve CI runner
         id: runner
         run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -126,6 +144,12 @@ jobs:
           python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py
           python3 greptimedb/tests/perf/test_ci_runner_scripts.py
           python3 greptimedb/tests/perf/test_agent_observability_summary.py
+      - name: Check runtime supports shared version-comparison data
+        if: inputs.compare_greptimedb
+        run: |
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
+          docker run --rm --entrypoint grep "$RUNTIME_IMAGE" -q -- '--data-root)' /opt/o11ybench/scripts/run-agent-observability.sh \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
       - name: Validate requested benchmark
         id: inputs
         shell: bash
@@ -289,6 +313,9 @@ jobs:
               victorialogs) resolve_image "$target" "victoriametrics/victoria-logs:$VICTORIALOGS_TAG" VICTORIALOGS_IMAGE ;;
             esac
           done
+          if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+            resolve_image greptimedb-baseline "greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG" BASELINE_GREPTIMEDB_IMAGE
+          fi
           jq -n --arg sha "$actual" --arg targets "$TARGETS" --arg profile "$PROFILE" \
             --arg cpus "$DB_CPUS" --arg memory "$DB_MEMORY" --argjson images "$images" \
             --arg provider "$PROVIDER" --arg instance_type "$INSTANCE_TYPE" --argjson disk_gib "$SYSTEM_DISK_GIB" \
@@ -312,6 +339,15 @@ jobs:
             --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
             --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
           cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
+      - name: Load and benchmark baseline GreptimeDB
+        id: baseline_greptimedb
+        if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+        run: |-
+          GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
+            --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
+            --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
+            --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
+            --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
       - name: Load and benchmark greptimedb
         id: greptimedb
         if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
@@ -357,6 +393,13 @@ jobs:
             GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
               --owner "$OWNER" --root "$ARTIFACT_ROOT"
           fi
+      - name: Clean up baseline containers
+        if: always() && inputs.compare_greptimedb
+        run: |-
+          if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
+            GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
+              --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
+          fi
       - name: Benchmark summary
         if: always()
         run: |
@@ -367,6 +410,16 @@ jobs:
         run: |
           python3 greptimedb/.github/scripts/agent-observability-summary.py \
             --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: Baseline load / lifecycle summary
+        if: always() && inputs.compare_greptimedb
+        run: |
+          echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+          python3 greptimedb/.github/scripts/agent-observability-summary.py \
+            --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      - name: GreptimeDB version comparison
+        if: always() && inputs.compare_greptimedb
+        run: |
+          python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
       - name: Upload results (no dataset)
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/tracesbench.yml

@@ -17,8 +17,18 @@ on:
         type: boolean
         default: false
         required: false
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
       greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
         type: string
         default: latest
         required: true
@@ -97,6 +107,8 @@ on:
 permissions:
   contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
   DATASET: ${{ inputs.dataset }}
   GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
   RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
@@ -150,6 +162,12 @@ jobs:
           [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
         fi
         printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
     - name: Resolve CI runner
       id: runner
       run: |
@@ -193,6 +211,10 @@ jobs:
         bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RUNTIME_IMAGE"
         metadata=$(docker image inspect "$RUNTIME_IMAGE")
         resolved=$(jq -er '.[0].RepoDigests[0]' <<< "$metadata")
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          docker run --rm "$resolved" scripts/run_tracesbench_docker.py --help | grep -q -- '--data-root' \
+            || { echo 'Comparison requires a runtime with --data-root support'; exit 1; }
+        fi
         revision=$(jq -er '.[0].Config.Labels["org.opencontainers.image.revision"]' <<< "$metadata")
         [[ "$revision" =~ ^[0-9a-f]{40}$ ]]
         # Refuse runtimes without the trace runner and its validated dataset/report contract.
@@ -323,12 +345,26 @@ jobs:
             printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
           fi
         done
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          image="greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG"
+          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
+          docker image inspect "$image" > "$METADATA_ROOT/greptimedb-baseline-image.json"
+          resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/greptimedb-baseline-image.json")
+          printf 'BASELINE_GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+        fi
     - name: Generate selected dataset once
       id: generate
       run: |
         python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
           --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
           --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Baseline GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
+          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
+          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
+          --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
     - name: GreptimeDB load, correctness and benchmark
       if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
       run: |
@@ -361,12 +397,27 @@ jobs:
         python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
           --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
           --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
+    - name: Clean up baseline containers
+      if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
+      run: |
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
+          --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
     - name: Benchmark summary
       if: always()
       run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
     - name: Load / lifecycle summary
       if: always()
       run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: Baseline load / lifecycle summary
+      if: always() && inputs.compare_greptimedb
+      run: |
+        echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
+        python3 greptimedb/.github/scripts/tracesbench-summary.py \
+          --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
     - name: Stage benchmark artifacts
       if: always()
       run: |
@@ -374,11 +425,15 @@ jobs:
         if [[ -d "$METADATA_ROOT" ]]; then
           rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
         fi
-        for entry in runs report report.json report.md; do
+        for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
           if [[ -e "$RUN_ROOT/$entry" ]]; then
             rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
           fi
         done
+        if [[ -d "$RUN_ROOT/baseline/runs" ]]; then
+          mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
+          rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
+        fi
     - name: Upload HTML and results (no corpus)
       if: always()
       uses: actions/upload-artifact@v4

.github/workflows/vmbench-long-range.yml

@@ -20,8 +20,18 @@ on:
         - 1mo
         default: 1w
         required: true
+      compare_greptimedb:
+        description: Compare baseline and candidate sequentially on the same host (adds one full GreptimeDB run)
+        type: boolean
+        default: false
+        required: false
+      baseline_greptimedb_tag:
+        description: Baseline Docker Hub greptime/greptimedb tag; required when comparison is enabled
+        type: string
+        default: v1.2.1
+        required: false
       greptimedb_tag:
-        description: Docker Hub greptime/greptimedb tag
+        description: Candidate Docker Hub greptime/greptimedb tag
         type: string
         default: latest
         required: true
@@ -79,6 +89,8 @@ on:
 permissions:
   contents: read
 env:
+  COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
+  BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
   BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
   RANGE_PRESET: ${{ inputs.range_preset }}
   GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
@@ -107,6 +119,12 @@ jobs:
       with:
         path: greptimedb
         persist-credentials: false
+    - name: Validate version comparison
+      env:
+        RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
+      run: |
+        python3 greptimedb/tests/perf/test_benchmark_version_comparison.py
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py validate
     - name: Resolve CI runner
       id: runner
       run: python3 greptimedb/.github/scripts/ci-runner-config.py --provider "$PROVIDER" --instance-type "$REQUESTED_INSTANCE_TYPE"
@@ -146,6 +164,9 @@ jobs:
         args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
           --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
           --run-classification controlled_server_benchmark --no-open-browser)
+        if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
+          args+=(--target "greptimedb:name=greptimedb-baseline,image=greptime/greptimedb:$BASELINE_GREPTIMEDB_TAG,url=http://127.0.0.1:14000/v1/prometheus/api/v1/query_range")
+        fi
         # Tags become target-spec fields; reject separators before passing them to the planner.
         for target in greptimedb victoriametrics; do
           case "$target" in
@@ -303,6 +324,10 @@ jobs:
       if: always()
       run: |
         docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
+    - name: GreptimeDB version comparison
+      if: always() && inputs.compare_greptimedb
+      run: |
+        python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
     - name: Stage benchmark artifacts
       if: always()
       run: |