ci: select tracesbench engines and GreptimeDB image version

Signed-off-by: WenyXu <wenymedia@gmail.com>
2026-10-26 21:56:17 +00:00 · 2026-09-29 09:07:52 +00:00
parent 23adb0b8e7
commit 6055ec3f17
3 changed files with 130 additions and 23 deletions
@@ -66,7 +66,7 @@ def load(root):
    return report, runs


-def benchmark(report, runs):
+def benchmark(report, runs, targets=TARGETS):
    lines = ['# Tracesbench benchmark summary', '',
             'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. '
             'Min/median/max come from the validated report, not recalculated samples. '
@@ -76,9 +76,11 @@ def benchmark(report, runs):
    statistics = {}
    for corpus in report.get('per_corpus', []):
        for stat in corpus.get('statistics', []):
+            if stat['target'] not in targets:
+                continue
            statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat
    rows, details = [], []
-    for target in TARGETS:
+    for target in targets:
        selected = [run for run in runs if run.get('target') == target]
        if not selected:
            lines.append(f'- **{target}**: unavailable / not run.')
@@ -117,9 +119,9 @@ def benchmark(report, runs):
    return '\n'.join(lines)


-def lifecycle(report, runs, root):
+def lifecycle(report, runs, root, targets=TARGETS):
    rows = []
-    for target in TARGETS:
+    for target in targets:
        selected = [r for r in runs if r.get('target') == target] or [{}]
        for run in selected:
            load = run.get('load', {})
@@ -145,16 +147,20 @@ def lifecycle(report, runs, root):
                              'DB CPU', 'DB memory', 'Running after load', 'OOM after load',
                              'Container removed'], rows), '',
                       render_resources([('dataset', root / 'runs' / 'generate')] +
-                                        [(t, root / 'runs' / t) for t in TARGETS], table)])
+                                        [(t, root / 'runs' / t) for t in targets], table)])


 def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('--root', type=Path, required=True)
    parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True)
+    parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines')
    args = parser.parse_args()
+    targets = args.targets.split(',')
+    if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets):
+        parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo')
    report, runs = load(args.root)
-    print(benchmark(report, runs) if args.section == 'benchmark' else lifecycle(report, runs, args.root))
+    print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets))


 if __name__ == '__main__':
@@ -2,6 +2,26 @@ name: Tracesbench
 on:
  workflow_dispatch:
    inputs:
+      run_greptimedb:
+        description: Run GreptimeDB
+        type: boolean
+        default: true
+        required: false
+      run_victoriatraces:
+        description: Run VictoriaTraces
+        type: boolean
+        default: false
+        required: false
+      run_tempo:
+        description: Run Tempo
+        type: boolean
+        default: false
+        required: false
+      greptimedb_tag:
+        description: Docker Hub greptime/greptimedb tag
+        type: string
+        default: latest
+        required: true
      dataset:
        description: Dataset profile (main is experimental; native queries may exceed memory limits)
        type: choice
@@ -11,7 +31,7 @@ on:
      runtime_image:
        description: Aliyun runtime image (tag or digest reference)
        type: string
-        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:4c6ff840d4e0@sha256:b7d0be034ba002768994a7187ce5c60fea4818537644f7a47d4913353b5c81ff
+        default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:36650fdb611a@sha256:e9babec6ca6b074e24a863e5fa30d985353875248e7222f23cd126bd4d52bddd
        required: true
      provider:
        description: Cloud provider for the benchmark runner
@@ -78,6 +98,7 @@ permissions:
  contents: read
 env:
  DATASET: ${{ inputs.dataset }}
+  GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
  RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
  RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
  BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
@@ -97,6 +118,10 @@ jobs:
    timeout-minutes: 15
    outputs:
      instance_type: ${{ steps.runner.outputs.instance_type }}
+      targets: ${{ steps.inputs.outputs.targets }}
+      greptimedb: ${{ steps.inputs.outputs.greptimedb }}
+      victoriatraces: ${{ steps.inputs.outputs.victoriatraces }}
+      tempo: ${{ steps.inputs.outputs.tempo }}
      system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }}
      runtime_image: ${{ steps.runtime.outputs.runtime_image }}
      o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }}
@@ -105,6 +130,26 @@ jobs:
      with:
        path: greptimedb
        persist-credentials: false
+    - name: Validate selected targets
+      id: inputs
+      env:
+        greptimedb: ${{ inputs.run_greptimedb }}
+        victoriatraces: ${{ inputs.run_victoriatraces }}
+        tempo: ${{ inputs.run_tempo }}
+      run: |
+        set -euo pipefail
+        selected=
+        for target in greptimedb victoriatraces tempo; do
+          enabled=${!target}
+          [[ "$enabled" == true || "$enabled" == false ]]
+          printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT"
+          if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi
+        done
+        [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; }
+        if [[ "$greptimedb" == true ]]; then
+          [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
+        fi
+        printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
    - name: Resolve CI runner
      id: runner
      run: |
@@ -157,6 +202,8 @@ jobs:
        grep -F -- '--runtime-memory' <<< "$help"
        grep -F -- '--query-timeout-seconds' <<< "$help"
        grep -F -- '--load-workers' <<< "$help"
+        grep -F -- '--targets' <<< "$help"
+        grep -F -- '--greptimedb-image' <<< "$help"
        docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py
        printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
  provision:
@@ -220,6 +267,7 @@ jobs:
    runs-on: ${{ needs.provision.outputs.label }}
    timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }}
    env:
+      TARGETS: ${{ needs.validate.outputs.targets }}
      RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }}
      RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }}
      RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }}
@@ -257,34 +305,46 @@ jobs:
        bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE"
        docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json"
        test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF"
-    # Preserve the pinned catalog images and require all three targets in the report.
    - name: Pull database images before generating data
      run: |
        set -euo pipefail
-        for target in greptimedb victoriatraces tempo; do
-          image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \
-            "o11ybench/workloads/tracesbench/catalogs/$target.json")
+        IFS=, read -ra targets <<< "$TARGETS"
+        for target in "${targets[@]}"; do
+          if [[ "$target" == greptimedb ]]; then
+            image="greptime/greptimedb:$GREPTIMEDB_TAG"
+          else
+            image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \
+              "o11ybench/workloads/tracesbench/catalogs/$target.json")
+          fi
          bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image"
          docker image inspect "$image" > "$METADATA_ROOT/$target-image.json"
+          if [[ "$target" == greptimedb ]]; then
+            resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json")
+            printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV"
+          fi
        done
    - name: Generate selected dataset once
+      id: generate
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: GreptimeDB load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
      run: |
-        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 \
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: VictoriaTraces load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \
          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Tempo load, correctness and benchmark
+      if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }}
      run: |
        python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \
          --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
@@ -292,7 +352,7 @@ jobs:
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Validate and render report
      run: |
-        python3 o11ybench/scripts/run_tracesbench_docker.py --stage report \
+        python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" \
          --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Clean up owned containers
@@ -303,10 +363,10 @@ jobs:
          --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
    - name: Benchmark summary
      if: always()
-      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --section benchmark >> "$GITHUB_STEP_SUMMARY"
+      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
    - name: Load / lifecycle summary
      if: always()
-      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
+      run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
    - name: Stage benchmark artifacts
      if: always()
      run: |
@@ -110,6 +110,35 @@ class ProviderConfigTest(unittest.TestCase):
        self.assertRegex(workflow, r"(?s)db_cpus:.*?default: '8'")
        self.assertRegex(workflow, r"(?s)db_memory:.*?default: 32g")

+    def test_traces_selected_targets_and_image_tag(self):
+        workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text()
+        step = workflow.split("    - name: Validate selected targets\n", 1)[1].split("    - name:", 1)[0]
+        command = textwrap.dedent(step.split("      run: |\n", 1)[1])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory) / "output"
+            for enabled, tag, expected in (
+                (("true", "false", "false"), "latest", "greptimedb"),
+                (("false", "true", "false"), "", "victoriatraces"),
+                (("false", "false", "true"), "", "tempo"),
+                (("true", "true", "true"), "v1.3.0-beta.1", "greptimedb,victoriatraces,tempo"),
+                (("false", "false", "false"), "latest", None),
+                (("true", "false", "false"), "latest; touch /tmp/unexpected", None),
+                (("true", "false", "false"), "", None),
+                (("invalid", "false", "false"), "latest", None),
+            ):
+                with self.subTest(enabled=enabled, tag=tag):
+                    output.write_text("")
+                    env = dict(os.environ, GREPTIMEDB_TAG=tag, GITHUB_OUTPUT=str(output))
+                    env.update(zip(("greptimedb", "victoriatraces", "tempo"), enabled))
+                    result = subprocess.run(["bash", "-c", command], env=env,
+                                            capture_output=True, text=True, timeout=5)
+                    if expected is None:
+                        self.assertNotEqual(result.returncode, 0)
+                        self.assertNotIn("targets=", output.read_text())
+                    else:
+                        self.assertEqual(result.returncode, 0, result.stderr)
+                        self.assertIn(f"targets={expected}\n", output.read_text())
+
    def test_wrong_provider_and_shell_fragments_rejected(self):
        for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"),
                                    ("Aliyun", "c7i.2xlarge"), ("AWS", ""),
@@ -520,14 +549,25 @@ class BenchmarkImagePullTest(unittest.TestCase):
        self.tool('docker', '''
            echo "$*" >> "$PULL_TEST_ROOT/docker.log"
            if [[ $1 == pull && $2 == ${PULL_FAIL_IMAGE:-} ]]; then exit 1; fi
-            if [[ $1 == image ]]; then echo '[]'; fi
+            if [[ $1 == image ]]; then printf '[{"RepoDigests":["greptime/greptimedb@sha256:%s"]}]\\n' "$(printf 'a%.0s' {1..64})"; fi
        ''')
-        self.env['METADATA_ROOT'] = str(self.root)
-        result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
-                                capture_output=True, text=True, timeout=5)
-        self.assertEqual(result.returncode, 0, result.stderr)
-        self.assertEqual(self.lines('docker'),
-                         [call for image in images for call in (f'pull {image}', f'image inspect {image}')])
+        images[0] = 'greptime/greptimedb:v1.3.0-beta.1'
+        self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1',
+                        GITHUB_ENV=str(self.root / 'env'))
+        for selected in (('greptimedb',), ('tempo',), ('greptimedb', 'victoriatraces', 'tempo')):
+            with self.subTest(selected=selected):
+                self.env['TARGETS'] = ','.join(selected)
+                (self.root / 'env').write_text('')
+                (self.root / 'docker.log').unlink(missing_ok=True)
+                result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
+                                        capture_output=True, text=True, timeout=5)
+                self.assertEqual(result.returncode, 0, result.stderr)
+                selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images)
+                                   if target in selected]
+                self.assertEqual(self.lines('docker'),
+                                 [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')])
+                expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n'
+                self.assertEqual((self.root / 'env').read_text(), expected_env if 'greptimedb' in selected else '')
        (self.root / 'docker.log').unlink()
        self.env['PULL_FAIL_IMAGE'] = images[1]
        result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env,
@@ -551,7 +591,8 @@ class BenchmarkImagePullTest(unittest.TestCase):
        end = traces.index('    - name: Generate selected dataset once')
        self.assertLess(start, end)
        step = traces[start:end]
-        self.assertIn('for target in greptimedb victoriatraces tempo', step)
+        self.assertIn('IFS=, read -ra targets <<< "$TARGETS"', step)
+        self.assertIn('for target in "${targets[@]}"', step)
        self.assertIn('o11ybench/workloads/tracesbench/catalogs/$target.json', step)
        self.assertIn('pull-benchmark-image.sh "$image"', step)
        self.assertIn('set -euo pipefail', step)