diff --git a/.github/scripts/tracesbench-summary.py b/.github/scripts/tracesbench-summary.py index 9351960d98a..78b33d7c212 100644 --- a/.github/scripts/tracesbench-summary.py +++ b/.github/scripts/tracesbench-summary.py @@ -66,7 +66,7 @@ def load(root): return report, runs -def benchmark(report, runs): +def benchmark(report, runs, targets=TARGETS): lines = ['# Tracesbench benchmark summary', '', 'Median Δ = (target / GreptimeDB − 1) × 100%; negative is faster. ' 'Min/median/max come from the validated report, not recalculated samples. ' @@ -76,9 +76,11 @@ def benchmark(report, runs): statistics = {} for corpus in report.get('per_corpus', []): for stat in corpus.get('statistics', []): + if stat['target'] not in targets: + continue statistics[(corpus['corpus'], stat['round'], stat['target'], stat['query_id'])] = stat rows, details = [], [] - for target in TARGETS: + for target in targets: selected = [run for run in runs if run.get('target') == target] if not selected: lines.append(f'- **{target}**: unavailable / not run.') @@ -117,9 +119,9 @@ def benchmark(report, runs): return '\n'.join(lines) -def lifecycle(report, runs, root): +def lifecycle(report, runs, root, targets=TARGETS): rows = [] - for target in TARGETS: + for target in targets: selected = [r for r in runs if r.get('target') == target] or [{}] for run in selected: load = run.get('load', {}) @@ -145,16 +147,20 @@ def lifecycle(report, runs, root): 'DB CPU', 'DB memory', 'Running after load', 'OOM after load', 'Container removed'], rows), '', render_resources([('dataset', root / 'runs' / 'generate')] + - [(t, root / 'runs' / t) for t in TARGETS], table)]) + [(t, root / 'runs' / t) for t in targets], table)]) def main(): parser = argparse.ArgumentParser() parser.add_argument('--root', type=Path, required=True) parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True) + parser.add_argument('--targets', default=','.join(TARGETS), help='Comma-separated selected engines') args = parser.parse_args() + targets = args.targets.split(',') + if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets): + parser.error('targets must be a non-empty comma-separated subset of greptimedb,victoriatraces,tempo') report, runs = load(args.root) - print(benchmark(report, runs) if args.section == 'benchmark' else lifecycle(report, runs, args.root)) + print(benchmark(report, runs, targets) if args.section == 'benchmark' else lifecycle(report, runs, args.root, targets)) if __name__ == '__main__': diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml index 6d4a11e9fa1..a9d85e330e3 100644 --- a/.github/workflows/tracesbench.yml +++ b/.github/workflows/tracesbench.yml @@ -2,6 +2,26 @@ name: Tracesbench on: workflow_dispatch: inputs: + run_greptimedb: + description: Run GreptimeDB + type: boolean + default: true + required: false + run_victoriatraces: + description: Run VictoriaTraces + type: boolean + default: false + required: false + run_tempo: + description: Run Tempo + type: boolean + default: false + required: false + greptimedb_tag: + description: Docker Hub greptime/greptimedb tag + type: string + default: latest + required: true dataset: description: Dataset profile (main is experimental; native queries may exceed memory limits) type: choice @@ -11,7 +31,7 @@ on: runtime_image: description: Aliyun runtime image (tag or digest reference) type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:4c6ff840d4e0@sha256:b7d0be034ba002768994a7187ce5c60fea4818537644f7a47d4913353b5c81ff + default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:36650fdb611a@sha256:e9babec6ca6b074e24a863e5fa30d985353875248e7222f23cd126bd4d52bddd required: true provider: description: Cloud provider for the benchmark runner @@ -78,6 +98,7 @@ permissions: contents: read env: DATASET: ${{ inputs.dataset }} + GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} RUNTIME_CPUS: ${{ inputs.runtime_cpus }} RUNTIME_MEMORY: ${{ inputs.runtime_memory }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} @@ -97,6 +118,10 @@ jobs: timeout-minutes: 15 outputs: instance_type: ${{ steps.runner.outputs.instance_type }} + targets: ${{ steps.inputs.outputs.targets }} + greptimedb: ${{ steps.inputs.outputs.greptimedb }} + victoriatraces: ${{ steps.inputs.outputs.victoriatraces }} + tempo: ${{ steps.inputs.outputs.tempo }} system_disk_gib: ${{ steps.runtime.outputs.system_disk_gib }} runtime_image: ${{ steps.runtime.outputs.runtime_image }} o11ybench_ref: ${{ steps.runtime.outputs.o11ybench_ref }} @@ -105,6 +130,26 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate selected targets + id: inputs + env: + greptimedb: ${{ inputs.run_greptimedb }} + victoriatraces: ${{ inputs.run_victoriatraces }} + tempo: ${{ inputs.run_tempo }} + run: | + set -euo pipefail + selected= + for target in greptimedb victoriatraces tempo; do + enabled=${!target} + [[ "$enabled" == true || "$enabled" == false ]] + printf '%s=%s\n' "$target" "$enabled" >> "$GITHUB_OUTPUT" + if [[ "$enabled" == true ]]; then selected="${selected:+$selected,}$target"; fi + done + [[ -n "$selected" ]] || { echo 'Select at least one database' >&2; exit 1; } + if [[ "$greptimedb" == true ]]; then + [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; } + fi + printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT" - name: Resolve CI runner id: runner run: | @@ -157,6 +202,8 @@ jobs: grep -F -- '--runtime-memory' <<< "$help" grep -F -- '--query-timeout-seconds' <<< "$help" grep -F -- '--load-workers' <<< "$help" + grep -F -- '--targets' <<< "$help" + grep -F -- '--greptimedb-image' <<< "$help" docker run --rm "${runtime_limits[@]}" "$resolved" scripts/check_run_tracesbench_docker.py printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT" provision: @@ -220,6 +267,7 @@ jobs: runs-on: ${{ needs.provision.outputs.label }} timeout-minutes: ${{ fromJSON(format('{0}', inputs.benchmark_timeout_minutes)) }} env: + TARGETS: ${{ needs.validate.outputs.targets }} RESOLVED_RUNTIME_IMAGE: ${{ needs.validate.outputs.runtime_image }} RESOLVED_O11YBENCH_REF: ${{ needs.validate.outputs.o11ybench_ref }} RUN_ROOT: ${{ inputs.benchmark_root }}/tracesbench-${{ github.run_id }}-${{ github.run_attempt }} @@ -257,34 +305,46 @@ jobs: bash greptimedb/.github/scripts/pull-benchmark-image.sh "$RESOLVED_RUNTIME_IMAGE" docker image inspect "$RESOLVED_RUNTIME_IMAGE" > "$METADATA_ROOT/runtime-image.json" test "$(jq -r '.[0].Config.Labels["org.opencontainers.image.revision"]' "$METADATA_ROOT/runtime-image.json")" = "$RESOLVED_O11YBENCH_REF" - # Preserve the pinned catalog images and require all three targets in the report. - name: Pull database images before generating data run: | set -euo pipefail - for target in greptimedb victoriatraces tempo; do - image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \ - "o11ybench/workloads/tracesbench/catalogs/$target.json") + IFS=, read -ra targets <<< "$TARGETS" + for target in "${targets[@]}"; do + if [[ "$target" == greptimedb ]]; then + image="greptime/greptimedb:$GREPTIMEDB_TAG" + else + image=$(jq -er '.target_identity.image | select(test("@sha256:[a-f0-9]{64}$"))' \ + "o11ybench/workloads/tracesbench/catalogs/$target.json") + fi bash greptimedb/.github/scripts/pull-benchmark-image.sh "$image" docker image inspect "$image" > "$METADATA_ROOT/$target-image.json" + if [[ "$target" == greptimedb ]]; then + resolved=$(jq -er '.[0].RepoDigests[0] | select(test("@sha256:[a-f0-9]{64}$"))' "$METADATA_ROOT/$target-image.json") + printf 'GREPTIMEDB_IMAGE=%s\n' "$resolved" >> "$GITHUB_ENV" + fi done - name: Generate selected dataset once + id: generate run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: GreptimeDB load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: VictoriaTraces load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }} run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target victoriatraces --port 19428 \ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Tempo load, correctness and benchmark + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.tempo == 'true' }} run: | python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target tempo --port 13200 \ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ @@ -292,7 +352,7 @@ jobs: --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Validate and render report run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage report \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Clean up owned containers @@ -303,10 +363,10 @@ jobs: --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Benchmark summary if: always() - run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --section benchmark >> "$GITHUB_STEP_SUMMARY" + run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" - name: Load / lifecycle summary if: always() - run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" - name: Stage benchmark artifacts if: always() run: | diff --git a/tests/perf/test_ci_runner_scripts.py b/tests/perf/test_ci_runner_scripts.py index 8300d8623b5..e1785955ca3 100644 --- a/tests/perf/test_ci_runner_scripts.py +++ b/tests/perf/test_ci_runner_scripts.py @@ -110,6 +110,35 @@ class ProviderConfigTest(unittest.TestCase): self.assertRegex(workflow, r"(?s)db_cpus:.*?default: '8'") self.assertRegex(workflow, r"(?s)db_memory:.*?default: 32g") + def test_traces_selected_targets_and_image_tag(self): + workflow = (SCRIPTS.parent / "workflows/tracesbench.yml").read_text() + step = workflow.split(" - name: Validate selected targets\n", 1)[1].split(" - name:", 1)[0] + command = textwrap.dedent(step.split(" run: |\n", 1)[1]) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "output" + for enabled, tag, expected in ( + (("true", "false", "false"), "latest", "greptimedb"), + (("false", "true", "false"), "", "victoriatraces"), + (("false", "false", "true"), "", "tempo"), + (("true", "true", "true"), "v1.3.0-beta.1", "greptimedb,victoriatraces,tempo"), + (("false", "false", "false"), "latest", None), + (("true", "false", "false"), "latest; touch /tmp/unexpected", None), + (("true", "false", "false"), "", None), + (("invalid", "false", "false"), "latest", None), + ): + with self.subTest(enabled=enabled, tag=tag): + output.write_text("") + env = dict(os.environ, GREPTIMEDB_TAG=tag, GITHUB_OUTPUT=str(output)) + env.update(zip(("greptimedb", "victoriatraces", "tempo"), enabled)) + result = subprocess.run(["bash", "-c", command], env=env, + capture_output=True, text=True, timeout=5) + if expected is None: + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("targets=", output.read_text()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"targets={expected}\n", output.read_text()) + def test_wrong_provider_and_shell_fragments_rejected(self): for provider, requested in (("ec2", "auto"), ("AWS", "ecs.c9i.2xlarge"), ("Aliyun", "c7i.2xlarge"), ("AWS", ""), @@ -520,14 +549,25 @@ class BenchmarkImagePullTest(unittest.TestCase): self.tool('docker', ''' echo "$*" >> "$PULL_TEST_ROOT/docker.log" if [[ $1 == pull && $2 == ${PULL_FAIL_IMAGE:-} ]]; then exit 1; fi - if [[ $1 == image ]]; then echo '[]'; fi + if [[ $1 == image ]]; then printf '[{"RepoDigests":["greptime/greptimedb@sha256:%s"]}]\\n' "$(printf 'a%.0s' {1..64})"; fi ''') - self.env['METADATA_ROOT'] = str(self.root) - result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env, - capture_output=True, text=True, timeout=5) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(self.lines('docker'), - [call for image in images for call in (f'pull {image}', f'image inspect {image}')]) + images[0] = 'greptime/greptimedb:v1.3.0-beta.1' + self.env.update(METADATA_ROOT=str(self.root), GREPTIMEDB_TAG='v1.3.0-beta.1', + GITHUB_ENV=str(self.root / 'env')) + for selected in (('greptimedb',), ('tempo',), ('greptimedb', 'victoriatraces', 'tempo')): + with self.subTest(selected=selected): + self.env['TARGETS'] = ','.join(selected) + (self.root / 'env').write_text('') + (self.root / 'docker.log').unlink(missing_ok=True) + result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env, + capture_output=True, text=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + selected_images = [image for target, image in zip(('greptimedb', 'victoriatraces', 'tempo'), images) + if target in selected] + self.assertEqual(self.lines('docker'), + [call for image in selected_images for call in (f'pull {image}', f'image inspect {image}')]) + expected_env = 'GREPTIMEDB_IMAGE=greptime/greptimedb@sha256:' + 'a' * 64 + '\n' + self.assertEqual((self.root / 'env').read_text(), expected_env if 'greptimedb' in selected else '') (self.root / 'docker.log').unlink() self.env['PULL_FAIL_IMAGE'] = images[1] result = subprocess.run(['bash', '-c', command], cwd=self.root, env=self.env, @@ -551,7 +591,8 @@ class BenchmarkImagePullTest(unittest.TestCase): end = traces.index(' - name: Generate selected dataset once') self.assertLess(start, end) step = traces[start:end] - self.assertIn('for target in greptimedb victoriatraces tempo', step) + self.assertIn('IFS=, read -ra targets <<< "$TARGETS"', step) + self.assertIn('for target in "${targets[@]}"', step) self.assertIn('o11ybench/workloads/tracesbench/catalogs/$target.json', step) self.assertIn('pull-benchmark-image.sh "$image"', step) self.assertIn('set -euo pipefail', step)