mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-06 19:06:58 +00:00
ci: compare GreptimeDB File and S3 in observability benchmarks (#10187)
* ci: compare GreptimeDB File and S3 in observability benchmarks Signed-off-by: WenyXu <wenymedia@gmail.com> * ci: separate File and S3 benchmark steps and verify report artifacts Signed-off-by: WenyXu <wenymedia@gmail.com> --------- Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
Weny Xu
@@ -133,9 +133,10 @@ def benchmark(root, targets):
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def lifecycle(root, targets):
|
||||
manifest = read(root / 'run-manifest.json')
|
||||
corpus = read(root / 'corpus' / 'summary.json')
|
||||
def lifecycle(root, targets, data_root=None):
|
||||
data_root = data_root or root
|
||||
manifest = read(data_root / 'run-manifest.json')
|
||||
corpus = read(data_root / 'corpus' / 'summary.json')
|
||||
lines = ['# Load / lifecycle summary', '',
|
||||
f'Dataset: **{cell(manifest.get("profile", corpus.get("data_profile", {}).get("id")))}**; rows: **{cell(corpus.get("line_count"))}**; '
|
||||
f'seed: `{cell(corpus.get("seed"))}`.',
|
||||
@@ -161,12 +162,13 @@ def lifecycle(root, targets):
|
||||
physical_rows = []
|
||||
load_rows = []
|
||||
details = ['<details>', '<summary>DDL, index configuration and image provenance</summary>', '']
|
||||
raw_bytes = read(root / 'corpus' / 'size.json').get('raw_jsonl_bytes')
|
||||
raw_bytes = read(data_root / 'corpus' / 'size.json').get('raw_jsonl_bytes')
|
||||
for t in targets:
|
||||
directory = result_dir(root, t)
|
||||
physical = read(directory / 'physical-evidence.json')
|
||||
storage = physical.get('storage', {})
|
||||
total = storage.get('engine_total_bytes')
|
||||
s3 = read(root / f'{t}-1' / 'storage.json').get('type') == 's3'
|
||||
total = None if s3 else storage.get('engine_total_bytes')
|
||||
stats = read(root / f'{t}-1' / 'post-load-docker-stats.json')
|
||||
ratio = f'{raw_bytes / total:.2f}×' if (physical.get('passed') is True and
|
||||
isinstance(raw_bytes, (int, float)) and raw_bytes > 0 and
|
||||
@@ -186,6 +188,7 @@ def lifecycle(root, targets):
|
||||
lines += ['## Load and physical evidence', '', table(['DB', 'Write transport', 'Batches', 'Load smoke rows'], load_rows), '',
|
||||
f'Raw JSONL size: **{size(raw_bytes)}**. Ratio = raw JSONL / engine bytes at collection time; '
|
||||
'not a settled-storage guarantee. CPU/memory are post-load snapshots, not peaks. '
|
||||
'S3 total/compression is omitted: local files exclude remote objects. '
|
||||
'Missing artifacts remain blank; pre-cleanup stats are not substituted.', '',
|
||||
table(['DB', 'Engine size', 'Size ratio', 'Materialization', 'SSTs / active parts',
|
||||
'Post-load CPU', 'Post-load memory / limit'], physical_rows), '', *details, '</details>', '']
|
||||
|
||||
@@ -62,11 +62,12 @@ def row(query, metric, old, new, compatible, reason):
|
||||
improvement_percent=delta, status='comparable' if delta is not None else reason)
|
||||
|
||||
|
||||
def agent(root):
|
||||
def agent(root, storage="file"):
|
||||
candidate, baseline = (root, root / "baseline") if storage == "file" else (root / "s3", root / "baseline/s3")
|
||||
def load(r):
|
||||
return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json')
|
||||
a, b = load(root / 'baseline'), load(root)
|
||||
finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)]
|
||||
a, b = load(baseline), load(candidate)
|
||||
finished = [r / 'greptimedb-1/exit-code.txt' for r in (baseline, candidate)]
|
||||
compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished)
|
||||
and a.get('passed') is True and b.get('passed') is True
|
||||
and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs')))
|
||||
@@ -81,8 +82,9 @@ def agent(root):
|
||||
return rows
|
||||
|
||||
|
||||
def traces(root):
|
||||
a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root))
|
||||
def traces(root, storage="file"):
|
||||
candidate, baseline = (root, root / "baseline") if storage == "file" else (root / "s3", root / "baseline/s3")
|
||||
a, b = (read(r / 'runs/greptimedb/run.json') for r in (baseline, candidate))
|
||||
compatible = (a.get('success') is True and b.get('success') is True
|
||||
and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus'))
|
||||
and same(a.get('execution', {}), b.get('execution', {}),
|
||||
@@ -110,18 +112,22 @@ def traces(root):
|
||||
return rows
|
||||
|
||||
|
||||
def long_range(root):
|
||||
def long_range(root, storage="file"):
|
||||
names = ("greptimedb-baseline", "greptimedb") if storage == "file" else ("greptimedb-s3-baseline", "greptimedb-s3")
|
||||
def target_root(name):
|
||||
nested = root / "o11ybench" / ("o11ybench-" + name)
|
||||
return nested if nested.exists() else root
|
||||
rows = []
|
||||
manifest = read(root / 'run-manifest.json')
|
||||
manifests = {name: read(target_root(name) / 'run-manifest.json') for name in names}
|
||||
healthy = all(
|
||||
all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results'
|
||||
else manifest.get(phase, {}).get(target, {})).get('state', '').split()
|
||||
all(field in (manifests[target].get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results'
|
||||
else manifests[target].get(phase, {}).get(target, {})).get('state', '').split()
|
||||
for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false'))
|
||||
for phase in ('post_load_results', 'post_query_results')
|
||||
for target in ('greptimedb-baseline', 'greptimedb'))
|
||||
for target in names)
|
||||
for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'):
|
||||
a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json'))
|
||||
for name in ('greptimedb-baseline', 'greptimedb'))
|
||||
a, b = (read(target_root(name) / name / 'results-vmbench-long-range' / (mode + '-summary.json'))
|
||||
for name in names)
|
||||
fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration',
|
||||
'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds')
|
||||
fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds')
|
||||
@@ -178,7 +184,13 @@ def main():
|
||||
parser.error('report requires --root and --kind')
|
||||
document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''),
|
||||
'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''),
|
||||
'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)}
|
||||
'rows': []}
|
||||
mode = os.environ.get('GREPTIMEDB_STORAGE', 'file')
|
||||
for storage in (['file', 's3'] if mode == 'both' else [mode]):
|
||||
rows = {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root, storage)
|
||||
for item in rows:
|
||||
item['query'] = storage.upper() + '/' + item['query']
|
||||
document['rows'].extend(rows)
|
||||
text = render(document)
|
||||
# Preserve partial comparisons even if a target failed before creating output.
|
||||
args.root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Select File/S3 benchmark arms without changing workload adapters."""
|
||||
import argparse
|
||||
import copy
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
|
||||
def validate(env):
|
||||
mode = env.get('GREPTIMEDB_STORAGE', 'file')
|
||||
if mode not in ('file', 's3', 'both'):
|
||||
raise ValueError('GREPTIMEDB_STORAGE must be file, s3 or both')
|
||||
if mode != 'file':
|
||||
if env.get('RUN_GREPTIMEDB') != 'true':
|
||||
raise ValueError('S3 requires Run GreptimeDB')
|
||||
if env.get('PROVIDER') != 'AWS':
|
||||
raise ValueError('S3 comparisons require an AWS runner')
|
||||
if not env.get('S3_BUCKET') or not env.get('S3_REGION'):
|
||||
raise ValueError('AWS_CI_TEST_BUCKET and AWS_CI_TEST_BUCKET_REGION must be configured')
|
||||
if env['S3_REGION'] != env.get('RUNNER_REGION'):
|
||||
raise ValueError('S3 bucket and EC2 runner regions must match')
|
||||
return mode
|
||||
|
||||
|
||||
def replace(command, key, value):
|
||||
result = list(command)
|
||||
if key in result:
|
||||
result[result.index(key) + 1] = value
|
||||
else:
|
||||
result += [key, value]
|
||||
return result
|
||||
|
||||
|
||||
def commands(command, mode):
|
||||
"""Recovery visits each owned root; execution uses explicit workflow steps."""
|
||||
stage = command[command.index('--stage') + 1]
|
||||
if stage != 'cleanup':
|
||||
raise ValueError('only cleanup may expand multiple arms')
|
||||
result = [list(command)]
|
||||
if mode != 'file':
|
||||
root = command[command.index('--root') + 1]
|
||||
owner = command[command.index('--owner') + 1]
|
||||
secondary = replace(command, '--root', root + '/s3')
|
||||
result.append(replace(secondary, '--owner', owner + '-s3'))
|
||||
return result
|
||||
|
||||
|
||||
def module(path):
|
||||
sys.path.insert(0, str(path.parent))
|
||||
spec = importlib.util.spec_from_file_location(path.stem.replace('-', '_'), path)
|
||||
result = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(result)
|
||||
return result
|
||||
|
||||
|
||||
def summary(kind, root, targets, section, mode):
|
||||
filename = 'agent-observability-summary.py' if kind == 'agent' else 'tracesbench-summary.py'
|
||||
renderer = module(Path(__file__).with_name(filename))
|
||||
primary = [t for t in targets if t != 'greptimedb' or mode != 's3']
|
||||
arms = [(root, primary, 'File')]
|
||||
if mode != 'file' and 'greptimedb' in targets:
|
||||
arms.append((root / 's3', ['greptimedb'], 'S3'))
|
||||
output = []
|
||||
for arm, selected, label in arms:
|
||||
if not selected:
|
||||
continue
|
||||
output.append('## ' + ('GreptimeDB S3' if label == 'S3' else 'Primary targets (GreptimeDB File)'))
|
||||
if kind == 'agent':
|
||||
body = (renderer.benchmark(arm, selected) if section == 'benchmark'
|
||||
else renderer.lifecycle(arm, selected, data_root=root.parent if root.name == 'baseline' else root))
|
||||
else:
|
||||
report, runs = renderer.load(arm)
|
||||
body = (renderer.benchmark(report, runs, selected) if section == 'benchmark'
|
||||
else renderer.lifecycle(report, runs, arm, selected))
|
||||
body = body.replace('| greptimedb |', '| GreptimeDB ' + label + ' |')
|
||||
body = body.replace('**greptimedb**', '**GreptimeDB ' + label + '**')
|
||||
output.append(body)
|
||||
if label == 'S3':
|
||||
directories = [arm / 'greptimedb-1'] if kind == 'agent' else [arm / 'runs/greptimedb']
|
||||
for directory in directories:
|
||||
for action in ('probe', 'inventory', 'cleanup'):
|
||||
path = directory / ('s3-' + action + '.json')
|
||||
data = json.loads(path.read_text()) if path.exists() else {}
|
||||
status = 'passed' if data.get('passed') is True else 'failed / unavailable'
|
||||
count = data.get('object_count', data.get('deleted_objects', '—'))
|
||||
output.append(f'- S3 {action}: **{status}**; objects: {count}.')
|
||||
return '\n\n'.join(output)
|
||||
|
||||
|
||||
def split_metrics(plan):
|
||||
"""One executor root per target, with the immutable dataset shared by all."""
|
||||
result = {}
|
||||
for target in plan['targets']:
|
||||
child = copy.deepcopy(plan)
|
||||
name = target['name']
|
||||
dest = str(Path(plan['output_root']) / 'o11ybench' / ('o11ybench-' + name))
|
||||
child['output_root'] = dest
|
||||
child['run_id'] = plan['run_id'] + '-' + name
|
||||
child['targets'] = [copy.deepcopy(target)]
|
||||
child['data']['generate'] = False
|
||||
child['report'].update(root=dest + '/report', index_html=dest + '/report/index.html')
|
||||
child['artifacts'] = {k: dest + '/' + Path(v).name for k, v in plan['artifacts'].items()}
|
||||
result[name] = child
|
||||
return result
|
||||
|
||||
|
||||
def metrics(action, args):
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--plan', type=Path, required=True)
|
||||
parser.add_argument('--tools', type=Path, required=True)
|
||||
parser.add_argument('--runtime-image', required=True)
|
||||
parser.add_argument('--target')
|
||||
a = parser.parse_args(args)
|
||||
plan = json.loads(a.plan.read_text())
|
||||
plan['runtime_image'] = a.runtime_image
|
||||
executor = module(a.tools / 'scripts/run_benchmark_plan.py')
|
||||
root = Path(plan['output_root'])
|
||||
if action == 'metrics-prepare':
|
||||
executor.require_safe_execute_root(root)
|
||||
root.mkdir(parents=True, exist_ok=False)
|
||||
manifest = executor.build_manifest(plan)
|
||||
executor.run_checked([str(executor.DOWNLOAD_DATA)], manifest,
|
||||
env=executor.generation_environment(plan))
|
||||
(root / 'generation.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
for name, child in split_metrics(plan).items():
|
||||
(a.plan.parent / (name + '.json')).write_text(json.dumps(child, indent=2) + '\n')
|
||||
elif action == 'metrics-run':
|
||||
path = a.plan.parent / (a.target + '.json')
|
||||
child = json.loads(path.read_text())
|
||||
executor.execute_plan(child, executor.build_manifest(child), replace=False)
|
||||
subprocess.run([sys.executable, str(a.tools / 'scripts/check_run_benchmark_plan.py'),
|
||||
'--artifacts', child['output_root']], check=True)
|
||||
else:
|
||||
# Validate every selected target together, retaining cross-target gates.
|
||||
manifests = [executor.build_manifest(child) for child in split_metrics(plan).values()]
|
||||
combined = executor.build_manifest(plan)
|
||||
combined['targets'] = [target for m in manifests for target in m['targets']]
|
||||
for command in executor.validation_commands(combined):
|
||||
subprocess.run(command, check=True)
|
||||
|
||||
|
||||
def overview(path):
|
||||
"""Render the same validated report data into the GitHub job summary."""
|
||||
data = json.loads(path.read_text())
|
||||
renderer = module(Path(__file__).with_name('agent-observability-summary.py'))
|
||||
lines = ['# Storage comparison', '', 'Smoke results are diagnostic; see classification per arm. '
|
||||
'No cold-object-store performance claim. Missing results are not passes.', '']
|
||||
for workload in data['workloads']:
|
||||
labels = {p['id']: p['label'] for p in workload['participants']}
|
||||
for scenario in workload['scenarios']:
|
||||
lines += ['## ' + workload['title'] + ' / ' + scenario['label'], '']
|
||||
rows = []
|
||||
for result in scenario['results']:
|
||||
lines += [labels[result['participant']] + ': ' + str(result.get('classification')), '']
|
||||
for query_id, query in result['queries'].items():
|
||||
m = query['metrics']
|
||||
rows.append([query_id, labels[result['participant']], query['status'],
|
||||
renderer.duration(m.get('p50_ms', m.get('median_ms'))),
|
||||
renderer.duration(m.get('p95_ms')), renderer.duration(m.get('p99_ms'))])
|
||||
lines.append(renderer.table(['Query', 'DB / storage', 'Status', 'P50 / median', 'P95', 'P99'], rows))
|
||||
return '\n'.join(lines)
|
||||
|
||||
|
||||
def audit(root, report, targets, mode, baseline=False):
|
||||
data = json.loads(report.read_text())
|
||||
actual = {(p['target'], p.get('storage', {}).get('type'), p['role'])
|
||||
for w in data['workloads'] for p in w['participants']}
|
||||
expected = {(t, None, 'comparison') for t in targets if t != 'greptimedb'}
|
||||
modes = ['file', 's3'] if mode == 'both' else [mode]
|
||||
if 'greptimedb' in targets:
|
||||
expected |= {('greptimedb', m, role) for m in modes
|
||||
for role in (['candidate', 'baseline'] if baseline else ['candidate'])}
|
||||
if actual != expected:
|
||||
raise ValueError(f'participant mismatch: expected {sorted(map(str, expected))}, got {sorted(map(str, actual))}')
|
||||
for w in data['workloads']:
|
||||
for scenario in w['scenarios']:
|
||||
for result in scenario['results']:
|
||||
queries = result['queries']
|
||||
if not any(q['status'] == 'passed' for q in queries.values()):
|
||||
raise ValueError('no successful measured queries: ' + result['participant'])
|
||||
if any(q['status'] in ('failed', 'unavailable') for q in queries.values()):
|
||||
raise ValueError('failed or unavailable queries: ' + result['participant'])
|
||||
for manifest_path in root.glob('o11ybench/*/run-manifest.json'):
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
if manifest.get('status') != 'passed':
|
||||
raise ValueError('incomplete target manifest: ' + str(manifest_path))
|
||||
probes = list(root.rglob('s3-probe.json'))
|
||||
expected_s3 = sum(m == 's3' for _, m, _ in expected)
|
||||
if len(probes) != expected_s3:
|
||||
raise ValueError('missing or duplicate S3 evidence')
|
||||
for probe in probes:
|
||||
for action in ('probe', 'inventory', 'cleanup'):
|
||||
payload = json.loads(probe.with_name('s3-' + action + '.json').read_text())
|
||||
if payload.get('passed') is not True or (action == 'inventory' and payload.get('object_count', 0) <= 0):
|
||||
raise ValueError('failed S3 ' + action)
|
||||
text = (root / 'ci-summary.md').read_text()
|
||||
for m in modes if 'greptimedb' in targets else []:
|
||||
if m.upper() not in text:
|
||||
raise ValueError('storage mode missing from saved summary: ' + m)
|
||||
result = dict(passed=True, participants=[dict(target=t, storage=m, role=role)
|
||||
for t, m, role in sorted(expected, key=str)], s3_arms=len(probes))
|
||||
(root / 'artifact-audit.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
lines = ['## Artifact validation', '', '- Selected participants: ' + str(len(expected)) + '; all present.',
|
||||
'- Query result coverage and S3 probe/inventory/cleanup: passed.', '']
|
||||
renderer = module(Path(__file__).with_name('agent-observability-summary.py'))
|
||||
rows = []
|
||||
for probe in probes:
|
||||
inv = json.loads(probe.with_name('s3-inventory.json').read_text())
|
||||
cleanup = json.loads(probe.with_name('s3-cleanup.json').read_text())
|
||||
rows.append([str(probe.parent.relative_to(root)), inv['object_count'], inv['object_bytes'],
|
||||
cleanup.get('deleted_objects'), 'passed'])
|
||||
if rows:
|
||||
lines.append(renderer.table(['S3 arm', 'Objects', 'Remote bytes', 'Deleted current objects', 'Cleanup'], rows))
|
||||
text = '\n'.join(lines) + '\n'
|
||||
with (root / 'ci-summary.md').open('a') as output:
|
||||
output.write(text)
|
||||
if os.environ.get('GITHUB_STEP_SUMMARY'):
|
||||
with Path(os.environ['GITHUB_STEP_SUMMARY']).open('a') as output:
|
||||
output.write(text)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'audit':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--root', type=Path, required=True)
|
||||
parser.add_argument('--report', type=Path, required=True)
|
||||
parser.add_argument('--targets', required=True)
|
||||
args = parser.parse_args(sys.argv[2:])
|
||||
print(json.dumps(audit(args.root, args.report, args.targets.split(','),
|
||||
os.environ.get('GREPTIMEDB_STORAGE', 'file'),
|
||||
os.environ.get('COMPARE_GREPTIMEDB') == 'true')))
|
||||
return
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'normalize-metrics':
|
||||
path = Path(sys.argv[2])
|
||||
plan = json.loads(path.read_text())
|
||||
for target in plan['targets']:
|
||||
if target.get('storage', {}).get('type') == 's3' and '-s3' not in target['name']:
|
||||
target['name'] = target['name'].replace('-baseline', '-s3-baseline') if target['name'].endswith('-baseline') else target['name'] + '-s3'
|
||||
path.write_text(json.dumps(plan, indent=2) + '\n')
|
||||
return
|
||||
if len(sys.argv) > 1 and sys.argv[1].startswith('metrics-'):
|
||||
metrics(sys.argv[1], sys.argv[2:])
|
||||
return
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'overview':
|
||||
print(overview(Path(sys.argv[2])))
|
||||
return
|
||||
if len(sys.argv) > 1 and sys.argv[1] == 'summary':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--kind', choices=['agent', 'traces'], required=True)
|
||||
parser.add_argument('--root', type=Path, required=True)
|
||||
parser.add_argument('--targets', required=True)
|
||||
parser.add_argument('--section', choices=['benchmark', 'lifecycle'], required=True)
|
||||
args = parser.parse_args(sys.argv[2:])
|
||||
print(summary(args.kind, args.root, args.targets.split(','), args.section,
|
||||
os.environ.get('GREPTIMEDB_STORAGE', 'file')))
|
||||
return
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=['validate', 'run'])
|
||||
parser.add_argument('command', nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
mode = validate(os.environ)
|
||||
if args.action == 'validate':
|
||||
print('GreptimeDB storage: ' + mode)
|
||||
return
|
||||
command = args.command[1:] if args.command[:1] == ['--'] else args.command
|
||||
planned = commands(command, mode)
|
||||
failed = 0
|
||||
for cmd in planned:
|
||||
result = subprocess.run(cmd)
|
||||
failed = failed or result.returncode
|
||||
# Cleanup must try both arms even if one failed. Target failures remain
|
||||
# visible, while the other arm can still produce diagnostic evidence.
|
||||
raise SystemExit(failed)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -2,6 +2,11 @@ name: Agent Observability Benchmark
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_storage:
|
||||
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
|
||||
type: choice
|
||||
options: [file, s3, both]
|
||||
default: file
|
||||
run_greptimedb:
|
||||
description: Run GreptimeDB
|
||||
type: boolean
|
||||
@@ -56,7 +61,7 @@ on:
|
||||
type: string
|
||||
required: false
|
||||
runtime_image:
|
||||
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
|
||||
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
|
||||
type: string
|
||||
default: auto
|
||||
required: true
|
||||
@@ -99,6 +104,11 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
|
||||
PROFILE: ${{ inputs.dataset }}
|
||||
@@ -106,7 +116,7 @@ env:
|
||||
CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }}
|
||||
VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }}
|
||||
O11YBENCH_REF: ${{ inputs.o11ybench_ref }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
@@ -130,6 +140,10 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Validate GreptimeDB storage
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_o11ybench_storage.py
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
|
||||
- name: Validate version comparison
|
||||
env:
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
@@ -370,24 +384,48 @@ jobs:
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
|
||||
cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json"
|
||||
- name: Load and benchmark baseline GreptimeDB
|
||||
id: baseline_greptimedb
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
|
||||
- name: Load and benchmark baseline GreptimeDB (File)
|
||||
id: baseline_greptimedb_file
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \
|
||||
--owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark greptimedb
|
||||
id: greptimedb
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
|
||||
- name: Load and benchmark baseline GreptimeDB (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
id: baseline_greptimedb_s3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \
|
||||
--owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline/s3" --data-root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark greptimedb (File)
|
||||
id: greptimedb_file
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \
|
||||
--owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark greptimedb (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
id: greptimedb_s3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }}
|
||||
run: |-
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \
|
||||
--owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/s3" --data-root "$ARTIFACT_ROOT" \
|
||||
--profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \
|
||||
--target greptimedb --image "$GREPTIMEDB_IMAGE" \
|
||||
--greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT"
|
||||
- name: Load and benchmark clickhouse
|
||||
id: clickhouse
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }}
|
||||
@@ -407,7 +445,7 @@ jobs:
|
||||
--target victorialogs --image "$VICTORIALOGS_IMAGE" \
|
||||
--victorialogs-http-port "$VICTORIALOGS_HTTP_PORT"
|
||||
- name: Compare all three targets
|
||||
if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
|
||||
if: ${{ !cancelled() && steps.greptimedb_file.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome
|
||||
== 'success' }}
|
||||
run: |-
|
||||
docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \
|
||||
@@ -418,45 +456,61 @@ jobs:
|
||||
--victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \
|
||||
--output /results/comparison.json
|
||||
- name: Clean up owned containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always()
|
||||
run: |-
|
||||
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
|
||||
GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--owner "$OWNER" --root "$ARTIFACT_ROOT"
|
||||
fi
|
||||
- name: Clean up baseline containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |-
|
||||
if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then
|
||||
GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \
|
||||
GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \
|
||||
--owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline"
|
||||
fi
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/agent-observability-summary.py \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Load / lifecycle summary
|
||||
if: always()
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/agent-observability-summary.py \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
|
||||
--root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Baseline load / lifecycle summary
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/agent-observability-summary.py \
|
||||
--root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '## Baseline GreptimeDB' | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \
|
||||
--root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: GreptimeDB version comparison
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Render storage comparison overview
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
python3 o11ybench/scripts/build_query_performance_report.py --agent "$ARTIFACT_ROOT" \
|
||||
--output "$ARTIFACT_ROOT-query-report.html" --json-output "$ARTIFACT_ROOT-query-report.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-query-report.json" \
|
||||
| tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT" \
|
||||
--report "$ARTIFACT_ROOT-query-report.json" --targets "$TARGETS"
|
||||
- name: Upload results (no dataset)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |-
|
||||
${{ env.ARTIFACT_ROOT }}-query-report.*
|
||||
${{ env.MANIFEST_PATH }}
|
||||
${{ env.ARTIFACT_ROOT }}
|
||||
!${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl
|
||||
|
||||
@@ -2,6 +2,11 @@ name: Tracesbench
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_storage:
|
||||
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
|
||||
type: choice
|
||||
options: [file, s3, both]
|
||||
default: file
|
||||
run_greptimedb:
|
||||
description: Run GreptimeDB
|
||||
type: boolean
|
||||
@@ -39,7 +44,7 @@ on:
|
||||
default: sanity
|
||||
required: true
|
||||
runtime_image:
|
||||
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
|
||||
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
|
||||
type: string
|
||||
default: auto
|
||||
required: true
|
||||
@@ -107,6 +112,11 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
|
||||
DATASET: ${{ inputs.dataset }}
|
||||
@@ -114,7 +124,7 @@ env:
|
||||
RUNTIME_CPUS: ${{ inputs.runtime_cpus }}
|
||||
RUNTIME_MEMORY: ${{ inputs.runtime_memory }}
|
||||
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
@@ -162,6 +172,10 @@ jobs:
|
||||
[[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; }
|
||||
fi
|
||||
printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT"
|
||||
- name: Validate GreptimeDB storage
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_o11ybench_storage.py
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
|
||||
- name: Validate version comparison
|
||||
env:
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
@@ -389,22 +403,44 @@ jobs:
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Baseline GreptimeDB load, correctness and benchmark
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }}
|
||||
- name: Baseline GreptimeDB load, correctness and benchmark (File)
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb"
|
||||
- name: GreptimeDB load, correctness and benchmark
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }}
|
||||
- name: Baseline GreptimeDB load, correctness and benchmark (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/baseline/s3" --data-root "$RUN_ROOT" --owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/s3/runs/greptimedb"
|
||||
- name: GreptimeDB load, correctness and benchmark (File)
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb"
|
||||
- name: GreptimeDB load, correctness and benchmark (S3)
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }}
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/s3/runs/greptimedb"
|
||||
- name: VictoriaTraces load, correctness and benchmark
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }}
|
||||
run: |
|
||||
@@ -422,49 +458,80 @@ jobs:
|
||||
- name: GreptimeDB version comparison
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Validate and render report
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Validate and render primary report
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
run: |
|
||||
if [[ "$GREPTIMEDB_STORAGE" == s3 ]]; then
|
||||
TARGETS=$(tr "," "\n" <<< "$TARGETS" | sed "/^greptimedb$/d" | paste -sd, -)
|
||||
[[ -n "$TARGETS" ]] || exit 0
|
||||
fi
|
||||
comparison_args=()
|
||||
if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
|
||||
if [[ "$COMPARE_GREPTIMEDB" == true && "$GREPTIMEDB_STORAGE" != s3 ]]; then
|
||||
comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json")
|
||||
fi
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "${TARGETS}" "${comparison_args[@]}" \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Validate and render S3 report
|
||||
if: always() && steps.generate.outcome == 'success' && inputs.greptimedb_storage != 'file'
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets greptimedb \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Clean up owned containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always() && steps.runtime.outcome == 'success'
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
--dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \
|
||||
--root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Clean up baseline containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \
|
||||
--root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Benchmark summary
|
||||
if: always()
|
||||
run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY"
|
||||
run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Load / lifecycle summary
|
||||
if: always()
|
||||
run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Baseline load / lifecycle summary
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/tracesbench-summary.py \
|
||||
--root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY"
|
||||
echo '## Baseline GreptimeDB' | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces \
|
||||
--root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Render storage comparison overview
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
mkdir -p "$RUN_ROOT/metadata"
|
||||
cp "$METADATA_ROOT"/*-image.json "$RUN_ROOT/metadata/"
|
||||
python3 o11ybench/scripts/build_query_performance_report.py --traces "$RUN_ROOT" \
|
||||
--output "$RUN_ROOT-query-report.html" --json-output "$RUN_ROOT-query-report.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$RUN_ROOT-query-report.json" \
|
||||
| tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$RUN_ROOT" \
|
||||
--report "$RUN_ROOT-query-report.json" --targets "$TARGETS"
|
||||
- name: Stage benchmark artifacts
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts"
|
||||
cp "$RUN_ROOT"-query-report.* "$RUNNER_TEMP/tracesbench-artifacts/" 2>/dev/null || true
|
||||
if [[ -d "$METADATA_ROOT" ]]; then
|
||||
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/"
|
||||
fi
|
||||
for entry in runs report report.json report.md version-comparison.json version-comparison.md; do
|
||||
for entry in artifact-audit.json metadata runs report report.json report.md ci-summary.md version-comparison.json version-comparison.md; do
|
||||
if [[ -e "$RUN_ROOT/$entry" ]]; then
|
||||
rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/"
|
||||
fi
|
||||
@@ -473,6 +540,12 @@ jobs:
|
||||
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline"
|
||||
rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/"
|
||||
fi
|
||||
for arm in s3 baseline/s3; do
|
||||
if [[ -d "$RUN_ROOT/$arm" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/$arm"
|
||||
rsync -a --no-owner --no-group --exclude scratch --exclude corpora "$RUN_ROOT/$arm/" "$RUNNER_TEMP/tracesbench-artifacts/$arm/"
|
||||
fi
|
||||
done
|
||||
- name: Upload HTML and results (no corpus)
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
|
||||
@@ -2,6 +2,11 @@ name: VMBench Long Range
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
greptimedb_storage:
|
||||
description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)'
|
||||
type: choice
|
||||
options: [file, s3, both]
|
||||
default: file
|
||||
run_greptimedb:
|
||||
description: Run GreptimeDB
|
||||
type: boolean
|
||||
@@ -41,7 +46,7 @@ on:
|
||||
default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344
|
||||
required: true
|
||||
runtime_image:
|
||||
description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides'
|
||||
description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides'
|
||||
type: string
|
||||
default: auto
|
||||
required: true
|
||||
@@ -89,13 +94,18 @@ on:
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }}
|
||||
S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
|
||||
S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
|
||||
RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }}
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }}
|
||||
BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }}
|
||||
BENCHMARK_ROOT: ${{ inputs.benchmark_root }}
|
||||
RANGE_PRESET: ${{ inputs.range_preset }}
|
||||
GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }}
|
||||
VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }}
|
||||
RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }}
|
||||
PROVIDER: ${{ inputs.provider }}
|
||||
REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }}
|
||||
SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }}
|
||||
@@ -119,6 +129,10 @@ jobs:
|
||||
with:
|
||||
path: greptimedb
|
||||
persist-credentials: false
|
||||
- name: Validate GreptimeDB storage
|
||||
run: |
|
||||
python3 greptimedb/tests/perf/test_o11ybench_storage.py
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py validate
|
||||
- name: Validate version comparison
|
||||
env:
|
||||
RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }}
|
||||
@@ -173,7 +187,7 @@ jobs:
|
||||
[[ "$revision" =~ ^[0-9a-f]{40}$ ]]
|
||||
printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT"
|
||||
printf 'run_id=long-range-%s-%s\nplan_artifact=vmbench-plan-%s-%s\n' "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT"
|
||||
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET"
|
||||
args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --greptimedb-storage "$GREPTIMEDB_STORAGE"
|
||||
--db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY"
|
||||
--run-classification controlled_server_benchmark --no-open-browser)
|
||||
if [[ "$COMPARE_GREPTIMEDB" == true ]]; then
|
||||
@@ -192,10 +206,11 @@ jobs:
|
||||
# The packaged planner owns profile, target and resource-limit validation.
|
||||
mkdir -p "$RUNNER_TEMP/vmbench-plans"
|
||||
# Keep the original warm defaults: serial 5/1 and c1/c4 30s/5s.
|
||||
docker run --rm "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
|
||||
docker run --rm -e S3_BUCKET -e S3_REGION "$resolved" scripts/o11ybench_plan.py "${args[@]}" \
|
||||
--run-id "long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
|
||||
--output-root "$BENCHMARK_ROOT/o11ybench/o11ybench-long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \
|
||||
--cache-policy warm > "$RUNNER_TEMP/vmbench-plans/warm.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py normalize-metrics "$RUNNER_TEMP/vmbench-plans/warm.json"
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: vmbench-plan-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
@@ -340,25 +355,87 @@ jobs:
|
||||
--plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --dry-run \
|
||||
> "$METADATA_ROOT/warm-execution-plan.json"
|
||||
cat "$METADATA_ROOT/warm-execution-plan.json"
|
||||
- name: Warm serial and concurrent benchmarks
|
||||
- name: Generate shared dataset and split target plans
|
||||
id: generate
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 -u o11ybench/scripts/run_benchmark_plan.py \
|
||||
--plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --execute \
|
||||
2>&1 | tee "$METADATA_ROOT/warm-execution.log"
|
||||
python3 o11ybench/scripts/check_run_benchmark_plan.py --artifacts "$ARTIFACT_ROOT-warm"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-prepare \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark Baseline GreptimeDB File
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 's3' }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-baseline \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark Baseline GreptimeDB S3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 'file' }}
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3-baseline \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark GreptimeDB File
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 's3' }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark GreptimeDB S3
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 'file' }}
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3 \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Load and benchmark VictoriaMetrics
|
||||
if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_victoriametrics }}
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target victoriametrics \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Validate all selected target results
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-validate \
|
||||
--plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE"
|
||||
- name: Clean up owned containers
|
||||
env:
|
||||
AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }}
|
||||
if: always()
|
||||
run: |
|
||||
docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv
|
||||
for name in greptimedb greptimedb-s3 greptimedb-baseline greptimedb-s3-baseline victoriametrics; do
|
||||
plan="$METADATA_ROOT/$name.json"
|
||||
[[ -f "$plan" ]] || continue
|
||||
run_id=$(jq -r '.run_id // empty' "$plan")
|
||||
[[ -n "$run_id" ]] || continue
|
||||
docker ps -aq --filter "label=o11ybench.run-id=$run_id" | xargs -r docker rm -fv
|
||||
done
|
||||
if [[ -d "$ARTIFACT_ROOT-warm" ]]; then
|
||||
for probe in "$ARTIFACT_ROOT-warm"/o11ybench/*/*/s3-probe.json; do
|
||||
[[ -f "$probe" ]] || continue
|
||||
python3 o11ybench/scripts/greptimedb_storage.py recover --runtime-image "$RESOLVED_RUNTIME_IMAGE" --output "${probe%/*}"
|
||||
done
|
||||
fi
|
||||
- name: GreptimeDB version comparison
|
||||
if: always() && inputs.compare_greptimedb
|
||||
run: |
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" | tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
- name: Render storage comparison overview
|
||||
if: always() && steps.generate.outcome == 'success'
|
||||
run: |
|
||||
mkdir -p "$ARTIFACT_ROOT-warm/metadata"
|
||||
cp "$METADATA_ROOT"/*-image.json "$ARTIFACT_ROOT-warm/metadata/"
|
||||
python3 o11ybench/scripts/build_query_performance_report.py --metrics "$ARTIFACT_ROOT-warm" \
|
||||
--output "$ARTIFACT_ROOT-warm-query-report.html" --json-output "$ARTIFACT_ROOT-warm-query-report.json"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-warm-query-report.json" \
|
||||
| tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY"
|
||||
python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT-warm" \
|
||||
--report "$ARTIFACT_ROOT-warm-query-report.json" --targets "$(jq -r '[.targets[].kind] | unique | join(",")' "$METADATA_ROOT/warm.json")"
|
||||
- name: Stage benchmark artifacts
|
||||
if: always()
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/vmbench-artifacts"
|
||||
cp "$ARTIFACT_ROOT-warm"-query-report.* "$RUNNER_TEMP/vmbench-artifacts/" 2>/dev/null || true
|
||||
if [[ -d "$METADATA_ROOT" ]]; then
|
||||
rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/vmbench-artifacts/"
|
||||
fi
|
||||
|
||||
@@ -625,7 +625,7 @@ class BenchmarkImagePullTest(unittest.TestCase):
|
||||
for reference in ("RUNTIME_IMAGE", "RESOLVED_RUNTIME_IMAGE", "reference"):
|
||||
self.assertIn(f'pull-benchmark-image.sh "${reference}"', long_range)
|
||||
self.assertLess(long_range.index('pull-benchmark-image.sh "$reference"'),
|
||||
long_range.index('- name: Warm serial and concurrent benchmarks'))
|
||||
long_range.index('- name: Generate shared dataset and split target plans'))
|
||||
self.assertIn('pull-benchmark-image.sh "$reference"', agent)
|
||||
self.assertLess(agent.index('resolve_image runtime'), agent.index('- name: Generate dataset once'))
|
||||
self.assertIn('pull-benchmark-image.sh "$RUNTIME_IMAGE"', traces)
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Offline contracts for File/S3 benchmark arm selection."""
|
||||
import importlib.util
|
||||
import copy
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
import unittest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
spec = importlib.util.spec_from_file_location('storage', ROOT / '.github/scripts/o11ybench-storage.py')
|
||||
storage = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(storage)
|
||||
|
||||
|
||||
class StorageTests(unittest.TestCase):
|
||||
def test_admission(self):
|
||||
self.assertEqual(storage.validate({}), 'file')
|
||||
env = dict(GREPTIMEDB_STORAGE='both', RUN_GREPTIMEDB='true', PROVIDER='AWS',
|
||||
S3_BUCKET='test', S3_REGION='us-east-1', RUNNER_REGION='us-east-1')
|
||||
self.assertEqual(storage.validate(env), 'both')
|
||||
for key, value in [('PROVIDER', 'Aliyun'), ('RUNNER_REGION', 'us-west-2'),
|
||||
('S3_BUCKET', ''), ('RUN_GREPTIMEDB', 'false'), ('GREPTIMEDB_STORAGE', 'invalid')]:
|
||||
with self.assertRaises(ValueError):
|
||||
storage.validate(dict(env, **{key: value}))
|
||||
def test_execution_must_not_expand_arms(self):
|
||||
command = ['runner', '--stage', 'target', '--root', '/run', '--owner', 'owner']
|
||||
with self.assertRaises(ValueError):
|
||||
storage.commands(command, 'both')
|
||||
|
||||
def test_metrics_split_preserves_data_and_target_contract(self):
|
||||
plan = dict(output_root='/tmp/o11ybench-test', run_id='test',
|
||||
data=dict(generate=True, data_home='/tmp/o11ybench-test/data'),
|
||||
targets=[dict(name='greptimedb', storage=dict(type='file')),
|
||||
dict(name='greptimedb-s3', storage=dict(type='s3', root='owned/prefix/'))],
|
||||
report=dict(root='old', index_html='old/index.html'),
|
||||
artifacts=dict(run_manifest='old/run-manifest.json'),
|
||||
query=dict(modes=['serial'], serial_tries=5))
|
||||
original = copy.deepcopy(plan)
|
||||
children = storage.split_metrics(plan)
|
||||
self.assertEqual(plan, original)
|
||||
for name, child in children.items():
|
||||
self.assertEqual(child['query'], plan['query'])
|
||||
self.assertEqual(child['data'], dict(plan['data'], generate=False))
|
||||
self.assertEqual(len(child['targets']), 1)
|
||||
self.assertEqual(child['targets'][0]['name'], name)
|
||||
self.assertEqual(child['output_root'], '/tmp/o11ybench-test/o11ybench/o11ybench-' + name)
|
||||
self.assertTrue(child['report']['root'].startswith(child['output_root']))
|
||||
self.assertNotEqual(children['greptimedb']['run_id'], children['greptimedb-s3']['run_id'])
|
||||
|
||||
def test_missing_s3_arm_is_not_a_success(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
report = root / 'report.json'
|
||||
report.write_text(json.dumps(dict(workloads=[dict(participants=[
|
||||
dict(target='greptimedb', role='candidate', storage=dict(type='file'))], scenarios=[])])))
|
||||
with self.assertRaisesRegex(ValueError, 'participant mismatch'):
|
||||
storage.audit(root, report, ['greptimedb'], 'both')
|
||||
|
||||
def test_s3_summary_is_explicit_when_missing(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
text = storage.summary('agent', Path(tmp), ['greptimedb'], 'benchmark', 'both')
|
||||
self.assertIn('GreptimeDB File', text)
|
||||
self.assertIn('GreptimeDB S3', text)
|
||||
self.assertIn('S3 cleanup: **failed / unavailable**', text)
|
||||
|
||||
def test_cleanup_both(self):
|
||||
command = ['runner', '--stage', 'cleanup', '--root', '/run', '--owner', 'owner']
|
||||
calls = storage.commands(command, 'both')
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertTrue(all('--greptimedb-storage' not in c for c in calls))
|
||||
def test_cleanup_s3_only_still_visits_primary_competitors(self):
|
||||
command = ['runner', '--stage', 'cleanup', '--root', '/run', '--owner', 'owner']
|
||||
calls = storage.commands(command, 's3')
|
||||
self.assertEqual(len(calls), 2)
|
||||
self.assertEqual(calls[1][calls[1].index('--root') + 1], '/run/s3')
|
||||
|
||||
def test_workflow_wiring(self):
|
||||
for name in ('agent-observability', 'tracesbench', 'vmbench-long-range'):
|
||||
text = (ROOT / '.github/workflows' / (name + '.yml')).read_text()
|
||||
self.assertIn('options: [file, s3, both]', text)
|
||||
self.assertIn('o11ybench-storage.py validate', text)
|
||||
self.assertIn('o11ybench-storage.py audit', text)
|
||||
self.assertIn('ci-summary.md', text)
|
||||
self.assertIn('AWS_SECRET_ACCESS_KEY:', text)
|
||||
self.assertNotIn('o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage target', text)
|
||||
if name == 'vmbench-long-range':
|
||||
self.assertIn('name: Load and benchmark GreptimeDB File', text)
|
||||
self.assertIn('name: Load and benchmark GreptimeDB S3', text)
|
||||
self.assertIn('metrics-prepare', text)
|
||||
self.assertIn('metrics-validate', text)
|
||||
else:
|
||||
self.assertIn('--stage target --greptimedb-storage file', text)
|
||||
self.assertIn('--stage target --greptimedb-storage s3', text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user