From b21fe455c93ab663968e79e270071e8fab0e181f Mon Sep 17 00:00:00 2001 From: WenyXu Date: Tue, 29 Sep 2026 08:45:29 +0000 Subject: [PATCH] ci: show benchmark resource usage and update parallel loader runtime Signed-off-by: WenyXu --- .../scripts/agent-observability-summary.py | 3 + .github/scripts/benchmark_resource_summary.py | 99 +++++++++++++++++++ .github/scripts/tracesbench-summary.py | 10 +- .github/workflows/agent-observability.yml | 2 +- .github/workflows/tracesbench.yml | 2 +- 5 files changed, 111 insertions(+), 5 deletions(-) create mode 100644 .github/scripts/benchmark_resource_summary.py diff --git a/.github/scripts/agent-observability-summary.py b/.github/scripts/agent-observability-summary.py index 38ba4e9dbdd..33ba3ad7e6e 100644 --- a/.github/scripts/agent-observability-summary.py +++ b/.github/scripts/agent-observability-summary.py @@ -18,6 +18,8 @@ import json import math from pathlib import Path +from benchmark_resource_summary import render_resources + TARGETS = ('greptimedb', 'clickhouse', 'victorialogs') @@ -201,6 +203,7 @@ def lifecycle(root, targets): rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else 'invalid timestamp units (legacy artifact)', entry.get('exit_code')]) lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), ''] + lines.append(render_resources([('dataset', root)] + [(t, root / f'{t}-1') for t in targets], table)) return '\n'.join(lines) diff --git a/.github/scripts/benchmark_resource_summary.py b/.github/scripts/benchmark_resource_summary.py new file mode 100644 index 00000000000..3a0a896c46e --- /dev/null +++ b/.github/scripts/benchmark_resource_summary.py @@ -0,0 +1,99 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Render shared O11yBench sampled resource artifacts for lifecycle summaries.""" +import json +import math + + +def number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value >= 0 + + +def metric(values, key, statistic): + entry = values.get(key, {}) + value = entry.get(statistic) + return value if number(entry.get('samples')) and entry['samples'] > 0 and number(value) else None + + +def decimal(value, suffix=''): + return f'{value:.2f}{suffix}' if number(value) else '—' + + +def memory(value): + if not number(value): + return '—' + for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'): + if value < 1024 or unit == 'TiB': + return decimal(value, f' {unit}') + value /= 1024 + + +def capacities(path): + # Quotas are metadata, not recomputed CPU statistics. Bound reads on long runs. + result = {} + try: + with path.open() as source: + for index, line in enumerate(source): + if index == 256: + break + record = json.loads(line) + cores = record.get('host', {}).get('cpu_count') + if number(cores) and cores > 0: + result.setdefault('host', cores) + for role, values in record.get('containers', {}).items(): + quota = values.get('configured_cpu_limit_cores') + if number(quota): + result.setdefault(role, quota) + if all(role in result for role in ('host', 'runtime', 'db')): + break + except (OSError, ValueError, TypeError, AttributeError): + pass + return result + + +def render_resources(scopes, table): + rows = [] + for scope, directory in scopes: + try: + summary = json.loads((directory / 'resources.summary.json').read_text()) + phases = summary.get('phases', {}) if summary.get('schema') == 'o11ybench-resources-v1' else {} + except (OSError, ValueError, TypeError, AttributeError): + phases = {} + if not isinstance(phases, dict) or not phases: + rows.append([scope, 'unavailable', *['—'] * 9]) + continue + limits = capacities(directory / 'resources.jsonl') + for phase, data in phases.items(): + for role in ('host', 'runtime', 'db'): + values = data.get('host', {}) if role == 'host' else data.get('containers', {}).get(role, {}) + if role == 'db' and scope == 'dataset': + continue + quota = limits.get(role) + cpu_samples = values.get('cpu_used_cores', {}).get('samples') + rows.append([scope, phase, role, cpu_samples, + decimal(metric(values, 'cpu_used_cores', 'mean')), + decimal(metric(values, 'cpu_used_cores', 'max')), + 'unlimited' if quota == 0 else decimal(quota), + memory(metric(values, 'memory_used_bytes' if role == 'host' else 'memory_usage_bytes', 'max')), + decimal(metric(values, 'io_wait_percent', 'mean'), '%'), + decimal(metric(values, 'io_wait_percent', 'max'), '%'), + memory(metric(values, 'swap_used_bytes', 'max'))]) + return '\n'.join(['## Sampled resources', '', + 'CPU is measured in cores. Mean/max summarize available samples, not continuous peaks; ' + 'short phases may have no CPU sample. CPU capacity is host cores or the configured container quota. ' + 'Host memory is total minus available; Docker memory excludes cache where supported. ' + 'Swap is host-wide usage, not swap activity. Missing evidence stays —.', '', + table(['Scope', 'Phase', 'Component', 'CPU samples', 'Mean CPU cores', 'Max CPU cores', + 'CPU capacity', 'Max memory', 'Mean I/O wait', 'Max I/O wait', 'Max swap'], rows), '']) diff --git a/.github/scripts/tracesbench-summary.py b/.github/scripts/tracesbench-summary.py index 1bbfa15a8dc..9351960d98a 100644 --- a/.github/scripts/tracesbench-summary.py +++ b/.github/scripts/tracesbench-summary.py @@ -18,6 +18,8 @@ import html import math from pathlib import Path +from benchmark_resource_summary import render_resources + TARGETS = ('greptimedb', 'victoriatraces', 'tempo') @@ -115,7 +117,7 @@ def benchmark(report, runs): return '\n'.join(lines) -def lifecycle(report, runs): +def lifecycle(report, runs, root): rows = [] for target in TARGETS: selected = [r for r in runs if r.get('target') == target] or [{}] @@ -141,7 +143,9 @@ def lifecycle(report, runs): table(['DB', 'Dataset', 'Loaded spans', 'Total', 'Ingest', 'Write success', 'Write errors', 'Load workers', 'Max in flight', 'DB CPU', 'DB memory', 'Running after load', 'OOM after load', - 'Container removed'], rows)]) + 'Container removed'], rows), '', + render_resources([('dataset', root / 'runs' / 'generate')] + + [(t, root / 'runs' / t) for t in TARGETS], table)]) def main(): @@ -150,7 +154,7 @@ def main(): parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True) args = parser.parse_args() report, runs = load(args.root) - print((benchmark if args.section == 'benchmark' else lifecycle)(report, runs)) + print(benchmark(report, runs) if args.section == 'benchmark' else lifecycle(report, runs, args.root)) if __name__ == '__main__': diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index 816d0dca7fa..fb457a4b080 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -48,7 +48,7 @@ on: runtime_image: description: Aliyun runtime image (tag or digest reference) type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:a78cc4971f90@sha256:22ea2820521ae18b4742c72b89f9e89f95fa9166ce7783a4b37a35e3cee8c8af + default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:4c6ff840d4e0@sha256:b7d0be034ba002768994a7187ce5c60fea4818537644f7a47d4913353b5c81ff required: true provider: description: Cloud provider for the benchmark runner diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml index ee9e2e986e0..6d4a11e9fa1 100644 --- a/.github/workflows/tracesbench.yml +++ b/.github/workflows/tracesbench.yml @@ -11,7 +11,7 @@ on: runtime_image: description: Aliyun runtime image (tag or digest reference) type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:e7a3945a6a83@sha256:d778f3e0a8cccc88d01e33eb9431a25fe54ac34a01a83aacb2bb682241bd1a27 + default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:4c6ff840d4e0@sha256:b7d0be034ba002768994a7187ce5c60fea4818537644f7a47d4913353b5c81ff required: true provider: description: Cloud provider for the benchmark runner