diff --git a/.github/scripts/agent-observability-summary.py b/.github/scripts/agent-observability-summary.py index 33ba3ad7e6e..29e3e995536 100644 --- a/.github/scripts/agent-observability-summary.py +++ b/.github/scripts/agent-observability-summary.py @@ -133,9 +133,10 @@ def benchmark(root, targets): return '\n'.join(lines) -def lifecycle(root, targets): - manifest = read(root / 'run-manifest.json') - corpus = read(root / 'corpus' / 'summary.json') +def lifecycle(root, targets, data_root=None): + data_root = data_root or root + manifest = read(data_root / 'run-manifest.json') + corpus = read(data_root / 'corpus' / 'summary.json') lines = ['# Load / lifecycle summary', '', f'Dataset: **{cell(manifest.get("profile", corpus.get("data_profile", {}).get("id")))}**; rows: **{cell(corpus.get("line_count"))}**; ' f'seed: `{cell(corpus.get("seed"))}`.', @@ -161,12 +162,13 @@ def lifecycle(root, targets): physical_rows = [] load_rows = [] details = ['
', 'DDL, index configuration and image provenance', ''] - raw_bytes = read(root / 'corpus' / 'size.json').get('raw_jsonl_bytes') + raw_bytes = read(data_root / 'corpus' / 'size.json').get('raw_jsonl_bytes') for t in targets: directory = result_dir(root, t) physical = read(directory / 'physical-evidence.json') storage = physical.get('storage', {}) - total = storage.get('engine_total_bytes') + s3 = read(root / f'{t}-1' / 'storage.json').get('type') == 's3' + total = None if s3 else storage.get('engine_total_bytes') stats = read(root / f'{t}-1' / 'post-load-docker-stats.json') ratio = f'{raw_bytes / total:.2f}×' if (physical.get('passed') is True and isinstance(raw_bytes, (int, float)) and raw_bytes > 0 and @@ -186,6 +188,7 @@ def lifecycle(root, targets): lines += ['## Load and physical evidence', '', table(['DB', 'Write transport', 'Batches', 'Load smoke rows'], load_rows), '', f'Raw JSONL size: **{size(raw_bytes)}**. Ratio = raw JSONL / engine bytes at collection time; ' 'not a settled-storage guarantee. CPU/memory are post-load snapshots, not peaks. ' + 'S3 total/compression is omitted: local files exclude remote objects. ' 'Missing artifacts remain blank; pre-cleanup stats are not substituted.', '', table(['DB', 'Engine size', 'Size ratio', 'Materialization', 'SSTs / active parts', 'Post-load CPU', 'Post-load memory / limit'], physical_rows), '', *details, '
', ''] diff --git a/.github/scripts/benchmark-version-comparison.py b/.github/scripts/benchmark-version-comparison.py index 1aa744df9ad..527797a2cb5 100644 --- a/.github/scripts/benchmark-version-comparison.py +++ b/.github/scripts/benchmark-version-comparison.py @@ -62,11 +62,12 @@ def row(query, metric, old, new, compatible, reason): improvement_percent=delta, status='comparable' if delta is not None else reason) -def agent(root): +def agent(root, storage="file"): + candidate, baseline = (root, root / "baseline") if storage == "file" else (root / "s3", root / "baseline/s3") def load(r): return read(r / 'greptimedb-1/greptimedb-1/measured-summary.json') - a, b = load(root / 'baseline'), load(root) - finished = [r / 'greptimedb-1/exit-code.txt' for r in (root / 'baseline', root)] + a, b = load(baseline), load(candidate) + finished = [r / 'greptimedb-1/exit-code.txt' for r in (baseline, candidate)] compatible = (all(p.is_file() and p.read_text().strip() == '0' for p in finished) and a.get('passed') is True and b.get('passed') is True and same(a, b, ('contract_sha256', 'data_profile', 'query_window', 'warmup_runs', 'measured_runs'))) @@ -81,8 +82,9 @@ def agent(root): return rows -def traces(root): - a, b = (read(r / 'runs/greptimedb/run.json') for r in (root / 'baseline', root)) +def traces(root, storage="file"): + candidate, baseline = (root, root / "baseline") if storage == "file" else (root / "s3", root / "baseline/s3") + a, b = (read(r / 'runs/greptimedb/run.json') for r in (baseline, candidate)) compatible = (a.get('success') is True and b.get('success') is True and same(a, b, ('expected', 'protocol', 'audit_expected', 'corpus')) and same(a.get('execution', {}), b.get('execution', {}), @@ -110,18 +112,22 @@ def traces(root): return rows -def long_range(root): +def long_range(root, storage="file"): + names = ("greptimedb-baseline", "greptimedb") if storage == "file" else ("greptimedb-s3-baseline", "greptimedb-s3") + def target_root(name): + nested = root / "o11ybench" / ("o11ybench-" + name) + return nested if nested.exists() else root rows = [] - manifest = read(root / 'run-manifest.json') + manifests = {name: read(target_root(name) / 'run-manifest.json') for name in names} healthy = all( - all(field in (manifest.get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results' - else manifest.get(phase, {}).get(target, {})).get('state', '').split() + all(field in (manifests[target].get(phase, {}).get(target, {}).get('docker', {}) if phase == 'post_load_results' + else manifests[target].get(phase, {}).get(target, {})).get('state', '').split() for field in ('Running=true', 'ExitCode=0', 'OOMKilled=false')) for phase in ('post_load_results', 'post_query_results') - for target in ('greptimedb-baseline', 'greptimedb')) + for target in names) for mode in ('serial', 'concurrent-duration-c1', 'concurrent-duration-c4'): - a, b = (read(root / name / 'results-vmbench-long-range' / (mode + '-summary.json')) - for name in ('greptimedb-baseline', 'greptimedb')) + a, b = (read(target_root(name) / name / 'results-vmbench-long-range' / (mode + '-summary.json')) + for name in names) fields = ('workload', 'query_file_sha256', 'start', 'end', 'step', 'profile', 'data_duration', 'scrape_interval', 'cache_policy_effective', 'mode', 'connection_mode', 'request_timeout_seconds') fields += ('tries', 'warmup_runs') if mode == 'serial' else ('concurrency', 'duration_seconds', 'warmup_seconds') @@ -178,7 +184,13 @@ def main(): parser.error('report requires --root and --kind') document = {'kind': args.kind, 'baseline_tag': os.environ.get('BASELINE_GREPTIMEDB_TAG', ''), 'candidate_tag': os.environ.get('GREPTIMEDB_TAG', ''), - 'rows': {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root)} + 'rows': []} + mode = os.environ.get('GREPTIMEDB_STORAGE', 'file') + for storage in (['file', 's3'] if mode == 'both' else [mode]): + rows = {'agent': agent, 'traces': traces, 'long-range': long_range}[args.kind](args.root, storage) + for item in rows: + item['query'] = storage.upper() + '/' + item['query'] + document['rows'].extend(rows) text = render(document) # Preserve partial comparisons even if a target failed before creating output. args.root.mkdir(parents=True, exist_ok=True) diff --git a/.github/scripts/o11ybench-storage.py b/.github/scripts/o11ybench-storage.py new file mode 100644 index 00000000000..2e982a245ef --- /dev/null +++ b/.github/scripts/o11ybench-storage.py @@ -0,0 +1,295 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Select File/S3 benchmark arms without changing workload adapters.""" +import argparse +import copy +import importlib.util +import json +from pathlib import Path +import sys +import os +import subprocess + + +def validate(env): + mode = env.get('GREPTIMEDB_STORAGE', 'file') + if mode not in ('file', 's3', 'both'): + raise ValueError('GREPTIMEDB_STORAGE must be file, s3 or both') + if mode != 'file': + if env.get('RUN_GREPTIMEDB') != 'true': + raise ValueError('S3 requires Run GreptimeDB') + if env.get('PROVIDER') != 'AWS': + raise ValueError('S3 comparisons require an AWS runner') + if not env.get('S3_BUCKET') or not env.get('S3_REGION'): + raise ValueError('AWS_CI_TEST_BUCKET and AWS_CI_TEST_BUCKET_REGION must be configured') + if env['S3_REGION'] != env.get('RUNNER_REGION'): + raise ValueError('S3 bucket and EC2 runner regions must match') + return mode + + +def replace(command, key, value): + result = list(command) + if key in result: + result[result.index(key) + 1] = value + else: + result += [key, value] + return result + + +def commands(command, mode): + """Recovery visits each owned root; execution uses explicit workflow steps.""" + stage = command[command.index('--stage') + 1] + if stage != 'cleanup': + raise ValueError('only cleanup may expand multiple arms') + result = [list(command)] + if mode != 'file': + root = command[command.index('--root') + 1] + owner = command[command.index('--owner') + 1] + secondary = replace(command, '--root', root + '/s3') + result.append(replace(secondary, '--owner', owner + '-s3')) + return result + + +def module(path): + sys.path.insert(0, str(path.parent)) + spec = importlib.util.spec_from_file_location(path.stem.replace('-', '_'), path) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +def summary(kind, root, targets, section, mode): + filename = 'agent-observability-summary.py' if kind == 'agent' else 'tracesbench-summary.py' + renderer = module(Path(__file__).with_name(filename)) + primary = [t for t in targets if t != 'greptimedb' or mode != 's3'] + arms = [(root, primary, 'File')] + if mode != 'file' and 'greptimedb' in targets: + arms.append((root / 's3', ['greptimedb'], 'S3')) + output = [] + for arm, selected, label in arms: + if not selected: + continue + output.append('## ' + ('GreptimeDB S3' if label == 'S3' else 'Primary targets (GreptimeDB File)')) + if kind == 'agent': + body = (renderer.benchmark(arm, selected) if section == 'benchmark' + else renderer.lifecycle(arm, selected, data_root=root.parent if root.name == 'baseline' else root)) + else: + report, runs = renderer.load(arm) + body = (renderer.benchmark(report, runs, selected) if section == 'benchmark' + else renderer.lifecycle(report, runs, arm, selected)) + body = body.replace('| greptimedb |', '| GreptimeDB ' + label + ' |') + body = body.replace('**greptimedb**', '**GreptimeDB ' + label + '**') + output.append(body) + if label == 'S3': + directories = [arm / 'greptimedb-1'] if kind == 'agent' else [arm / 'runs/greptimedb'] + for directory in directories: + for action in ('probe', 'inventory', 'cleanup'): + path = directory / ('s3-' + action + '.json') + data = json.loads(path.read_text()) if path.exists() else {} + status = 'passed' if data.get('passed') is True else 'failed / unavailable' + count = data.get('object_count', data.get('deleted_objects', '—')) + output.append(f'- S3 {action}: **{status}**; objects: {count}.') + return '\n\n'.join(output) + + +def split_metrics(plan): + """One executor root per target, with the immutable dataset shared by all.""" + result = {} + for target in plan['targets']: + child = copy.deepcopy(plan) + name = target['name'] + dest = str(Path(plan['output_root']) / 'o11ybench' / ('o11ybench-' + name)) + child['output_root'] = dest + child['run_id'] = plan['run_id'] + '-' + name + child['targets'] = [copy.deepcopy(target)] + child['data']['generate'] = False + child['report'].update(root=dest + '/report', index_html=dest + '/report/index.html') + child['artifacts'] = {k: dest + '/' + Path(v).name for k, v in plan['artifacts'].items()} + result[name] = child + return result + + +def metrics(action, args): + parser = argparse.ArgumentParser() + parser.add_argument('--plan', type=Path, required=True) + parser.add_argument('--tools', type=Path, required=True) + parser.add_argument('--runtime-image', required=True) + parser.add_argument('--target') + a = parser.parse_args(args) + plan = json.loads(a.plan.read_text()) + plan['runtime_image'] = a.runtime_image + executor = module(a.tools / 'scripts/run_benchmark_plan.py') + root = Path(plan['output_root']) + if action == 'metrics-prepare': + executor.require_safe_execute_root(root) + root.mkdir(parents=True, exist_ok=False) + manifest = executor.build_manifest(plan) + executor.run_checked([str(executor.DOWNLOAD_DATA)], manifest, + env=executor.generation_environment(plan)) + (root / 'generation.json').write_text(json.dumps(manifest, indent=2) + '\n') + for name, child in split_metrics(plan).items(): + (a.plan.parent / (name + '.json')).write_text(json.dumps(child, indent=2) + '\n') + elif action == 'metrics-run': + path = a.plan.parent / (a.target + '.json') + child = json.loads(path.read_text()) + executor.execute_plan(child, executor.build_manifest(child), replace=False) + subprocess.run([sys.executable, str(a.tools / 'scripts/check_run_benchmark_plan.py'), + '--artifacts', child['output_root']], check=True) + else: + # Validate every selected target together, retaining cross-target gates. + manifests = [executor.build_manifest(child) for child in split_metrics(plan).values()] + combined = executor.build_manifest(plan) + combined['targets'] = [target for m in manifests for target in m['targets']] + for command in executor.validation_commands(combined): + subprocess.run(command, check=True) + + +def overview(path): + """Render the same validated report data into the GitHub job summary.""" + data = json.loads(path.read_text()) + renderer = module(Path(__file__).with_name('agent-observability-summary.py')) + lines = ['# Storage comparison', '', 'Smoke results are diagnostic; see classification per arm. ' + 'No cold-object-store performance claim. Missing results are not passes.', ''] + for workload in data['workloads']: + labels = {p['id']: p['label'] for p in workload['participants']} + for scenario in workload['scenarios']: + lines += ['## ' + workload['title'] + ' / ' + scenario['label'], ''] + rows = [] + for result in scenario['results']: + lines += [labels[result['participant']] + ': ' + str(result.get('classification')), ''] + for query_id, query in result['queries'].items(): + m = query['metrics'] + rows.append([query_id, labels[result['participant']], query['status'], + renderer.duration(m.get('p50_ms', m.get('median_ms'))), + renderer.duration(m.get('p95_ms')), renderer.duration(m.get('p99_ms'))]) + lines.append(renderer.table(['Query', 'DB / storage', 'Status', 'P50 / median', 'P95', 'P99'], rows)) + return '\n'.join(lines) + + +def audit(root, report, targets, mode, baseline=False): + data = json.loads(report.read_text()) + actual = {(p['target'], p.get('storage', {}).get('type'), p['role']) + for w in data['workloads'] for p in w['participants']} + expected = {(t, None, 'comparison') for t in targets if t != 'greptimedb'} + modes = ['file', 's3'] if mode == 'both' else [mode] + if 'greptimedb' in targets: + expected |= {('greptimedb', m, role) for m in modes + for role in (['candidate', 'baseline'] if baseline else ['candidate'])} + if actual != expected: + raise ValueError(f'participant mismatch: expected {sorted(map(str, expected))}, got {sorted(map(str, actual))}') + for w in data['workloads']: + for scenario in w['scenarios']: + for result in scenario['results']: + queries = result['queries'] + if not any(q['status'] == 'passed' for q in queries.values()): + raise ValueError('no successful measured queries: ' + result['participant']) + if any(q['status'] in ('failed', 'unavailable') for q in queries.values()): + raise ValueError('failed or unavailable queries: ' + result['participant']) + for manifest_path in root.glob('o11ybench/*/run-manifest.json'): + manifest = json.loads(manifest_path.read_text()) + if manifest.get('status') != 'passed': + raise ValueError('incomplete target manifest: ' + str(manifest_path)) + probes = list(root.rglob('s3-probe.json')) + expected_s3 = sum(m == 's3' for _, m, _ in expected) + if len(probes) != expected_s3: + raise ValueError('missing or duplicate S3 evidence') + for probe in probes: + for action in ('probe', 'inventory', 'cleanup'): + payload = json.loads(probe.with_name('s3-' + action + '.json').read_text()) + if payload.get('passed') is not True or (action == 'inventory' and payload.get('object_count', 0) <= 0): + raise ValueError('failed S3 ' + action) + text = (root / 'ci-summary.md').read_text() + for m in modes if 'greptimedb' in targets else []: + if m.upper() not in text: + raise ValueError('storage mode missing from saved summary: ' + m) + result = dict(passed=True, participants=[dict(target=t, storage=m, role=role) + for t, m, role in sorted(expected, key=str)], s3_arms=len(probes)) + (root / 'artifact-audit.json').write_text(json.dumps(result, indent=2) + '\n') + lines = ['## Artifact validation', '', '- Selected participants: ' + str(len(expected)) + '; all present.', + '- Query result coverage and S3 probe/inventory/cleanup: passed.', ''] + renderer = module(Path(__file__).with_name('agent-observability-summary.py')) + rows = [] + for probe in probes: + inv = json.loads(probe.with_name('s3-inventory.json').read_text()) + cleanup = json.loads(probe.with_name('s3-cleanup.json').read_text()) + rows.append([str(probe.parent.relative_to(root)), inv['object_count'], inv['object_bytes'], + cleanup.get('deleted_objects'), 'passed']) + if rows: + lines.append(renderer.table(['S3 arm', 'Objects', 'Remote bytes', 'Deleted current objects', 'Cleanup'], rows)) + text = '\n'.join(lines) + '\n' + with (root / 'ci-summary.md').open('a') as output: + output.write(text) + if os.environ.get('GITHUB_STEP_SUMMARY'): + with Path(os.environ['GITHUB_STEP_SUMMARY']).open('a') as output: + output.write(text) + return result + + +def main(): + if len(sys.argv) > 1 and sys.argv[1] == 'audit': + parser = argparse.ArgumentParser() + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--report', type=Path, required=True) + parser.add_argument('--targets', required=True) + args = parser.parse_args(sys.argv[2:]) + print(json.dumps(audit(args.root, args.report, args.targets.split(','), + os.environ.get('GREPTIMEDB_STORAGE', 'file'), + os.environ.get('COMPARE_GREPTIMEDB') == 'true'))) + return + if len(sys.argv) > 1 and sys.argv[1] == 'normalize-metrics': + path = Path(sys.argv[2]) + plan = json.loads(path.read_text()) + for target in plan['targets']: + if target.get('storage', {}).get('type') == 's3' and '-s3' not in target['name']: + target['name'] = target['name'].replace('-baseline', '-s3-baseline') if target['name'].endswith('-baseline') else target['name'] + '-s3' + path.write_text(json.dumps(plan, indent=2) + '\n') + return + if len(sys.argv) > 1 and sys.argv[1].startswith('metrics-'): + metrics(sys.argv[1], sys.argv[2:]) + return + if len(sys.argv) > 1 and sys.argv[1] == 'overview': + print(overview(Path(sys.argv[2]))) + return + if len(sys.argv) > 1 and sys.argv[1] == 'summary': + parser = argparse.ArgumentParser() + parser.add_argument('--kind', choices=['agent', 'traces'], required=True) + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--targets', required=True) + parser.add_argument('--section', choices=['benchmark', 'lifecycle'], required=True) + args = parser.parse_args(sys.argv[2:]) + print(summary(args.kind, args.root, args.targets.split(','), args.section, + os.environ.get('GREPTIMEDB_STORAGE', 'file'))) + return + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=['validate', 'run']) + parser.add_argument('command', nargs=argparse.REMAINDER) + args = parser.parse_args() + mode = validate(os.environ) + if args.action == 'validate': + print('GreptimeDB storage: ' + mode) + return + command = args.command[1:] if args.command[:1] == ['--'] else args.command + planned = commands(command, mode) + failed = 0 + for cmd in planned: + result = subprocess.run(cmd) + failed = failed or result.returncode + # Cleanup must try both arms even if one failed. Target failures remain + # visible, while the other arm can still produce diagnostic evidence. + raise SystemExit(failed) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index 0b4f8f6a4f9..055af1d0f26 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -2,6 +2,11 @@ name: Agent Observability Benchmark on: workflow_dispatch: inputs: + greptimedb_storage: + description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)' + type: choice + options: [file, s3, both] + default: file run_greptimedb: description: Run GreptimeDB type: boolean @@ -56,7 +61,7 @@ on: type: string required: false runtime_image: - description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' + description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides' type: string default: auto required: true @@ -99,6 +104,11 @@ on: permissions: contents: read env: + GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }} + S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }} + S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }} + RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }} + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} PROFILE: ${{ inputs.dataset }} @@ -106,7 +116,7 @@ env: CLICKHOUSE_TAG: ${{ inputs.clickhouse_tag }} VICTORIALOGS_TAG: ${{ inputs.victorialogs_tag }} O11YBENCH_REF: ${{ inputs.o11ybench_ref }} - RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:2f40fb3fdbdc@sha256:8940782216bf5d066308fd71f6e346d950073c41cfb5756c5f3dc8aacf069cbc' }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -130,6 +140,10 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate GreptimeDB storage + run: | + python3 greptimedb/tests/perf/test_o11ybench_storage.py + python3 greptimedb/.github/scripts/o11ybench-storage.py validate - name: Validate version comparison env: RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} @@ -370,24 +384,48 @@ jobs: --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json" - - name: Load and benchmark baseline GreptimeDB - id: baseline_greptimedb - if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} + - name: Load and benchmark baseline GreptimeDB (File) + id: baseline_greptimedb_file + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }} run: |- - GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \ --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline" --data-root "$ARTIFACT_ROOT" \ --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \ --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" - - name: Load and benchmark greptimedb - id: greptimedb - if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} + - name: Load and benchmark baseline GreptimeDB (S3) + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} + id: baseline_greptimedb_s3 + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }} run: |- - GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \ + --owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/baseline/s3" --data-root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target greptimedb --image "$BASELINE_GREPTIMEDB_IMAGE" \ + --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" + - name: Load and benchmark greptimedb (File) + id: greptimedb_file + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }} + run: |- + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage file --execute \ --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ --target greptimedb --image "$GREPTIMEDB_IMAGE" \ --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" + - name: Load and benchmark greptimedb (S3) + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} + id: greptimedb_s3 + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }} + run: |- + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --greptimedb-storage s3 --execute \ + --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT/s3" --data-root "$ARTIFACT_ROOT" \ + --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ + --target greptimedb --image "$GREPTIMEDB_IMAGE" \ + --greptimedb-http-port "$GREPTIMEDB_HTTP_PORT" --greptimedb-grpc-port "$GREPTIMEDB_GRPC_PORT" - name: Load and benchmark clickhouse id: clickhouse if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }} @@ -407,7 +445,7 @@ jobs: --target victorialogs --image "$VICTORIALOGS_IMAGE" \ --victorialogs-http-port "$VICTORIALOGS_HTTP_PORT" - name: Compare all three targets - if: ${{ !cancelled() && steps.greptimedb.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome + if: ${{ !cancelled() && steps.greptimedb_file.outcome == 'success' && steps.clickhouse.outcome == 'success' && steps.victorialogs.outcome == 'success' }} run: |- docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \ @@ -418,45 +456,61 @@ jobs: --victorialogs-summary /results/victorialogs-1/victorialogs-1/measured-summary.json \ --output /results/comparison.json - name: Clean up owned containers + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} if: always() run: |- if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then - GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --owner "$OWNER" --root "$ARTIFACT_ROOT" fi - name: Clean up baseline containers + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} if: always() && inputs.compare_greptimedb run: |- if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then - GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + GITHUB_STEP_SUMMARY= python3 greptimedb/.github/scripts/o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --owner "$OWNER-base" --root "$ARTIFACT_ROOT/baseline" fi - name: Benchmark summary if: always() run: | - python3 greptimedb/.github/scripts/agent-observability-summary.py \ - --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \ + --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" - name: Load / lifecycle summary if: always() run: | - python3 greptimedb/.github/scripts/agent-observability-summary.py \ - --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \ + --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" - name: Baseline load / lifecycle summary if: always() && inputs.compare_greptimedb run: | - echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" - python3 greptimedb/.github/scripts/agent-observability-summary.py \ - --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" + echo '## Baseline GreptimeDB' | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind agent \ + --root "$ARTIFACT_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" - name: GreptimeDB version comparison if: always() && inputs.compare_greptimedb run: | - python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind agent --root "$ARTIFACT_ROOT" | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Render storage comparison overview + if: always() && steps.generate.outcome == 'success' + run: | + python3 o11ybench/scripts/build_query_performance_report.py --agent "$ARTIFACT_ROOT" \ + --output "$ARTIFACT_ROOT-query-report.html" --json-output "$ARTIFACT_ROOT-query-report.json" + python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-query-report.json" \ + | tee -a "$ARTIFACT_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT" \ + --report "$ARTIFACT_ROOT-query-report.json" --targets "$TARGETS" - name: Upload results (no dataset) if: always() uses: actions/upload-artifact@v4 with: name: agent-observability-${{ github.run_id }}-${{ github.run_attempt }} path: |- + ${{ env.ARTIFACT_ROOT }}-query-report.* ${{ env.MANIFEST_PATH }} ${{ env.ARTIFACT_ROOT }} !${{ env.ARTIFACT_ROOT }}/corpus/agent_observations.jsonl diff --git a/.github/workflows/tracesbench.yml b/.github/workflows/tracesbench.yml index cdd02e1846f..daedeb38826 100644 --- a/.github/workflows/tracesbench.yml +++ b/.github/workflows/tracesbench.yml @@ -2,6 +2,11 @@ name: Tracesbench on: workflow_dispatch: inputs: + greptimedb_storage: + description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)' + type: choice + options: [file, s3, both] + default: file run_greptimedb: description: Run GreptimeDB type: boolean @@ -39,7 +44,7 @@ on: default: sanity required: true runtime_image: - description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' + description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides' type: string default: auto required: true @@ -107,6 +112,11 @@ on: permissions: contents: read env: + GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }} + S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }} + S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }} + RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }} + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} DATASET: ${{ inputs.dataset }} @@ -114,7 +124,7 @@ env: RUNTIME_CPUS: ${{ inputs.runtime_cpus }} RUNTIME_MEMORY: ${{ inputs.runtime_memory }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} - RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:77a84e1c8136@sha256:c482993ec177b026235d6cd0c61d4fb6d6f1088bc78bdc0e5964a5398de6dba0' }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -162,6 +172,10 @@ jobs: [[ "$GREPTIMEDB_TAG" =~ ^[a-zA-Z0-9_][a-zA-Z0-9_.-]{0,127}$ ]] || { echo 'Invalid GreptimeDB image tag' >&2; exit 1; } fi printf 'targets=%s\n' "$selected" >> "$GITHUB_OUTPUT" + - name: Validate GreptimeDB storage + run: | + python3 greptimedb/tests/perf/test_o11ybench_storage.py + python3 greptimedb/.github/scripts/o11ybench-storage.py validate - name: Validate version comparison env: RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} @@ -389,22 +403,44 @@ jobs: python3 o11ybench/scripts/run_tracesbench_docker.py --stage generate \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - - name: Baseline GreptimeDB load, correctness and benchmark - if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.compare_greptimedb }} + - name: Baseline GreptimeDB load, correctness and benchmark (File) + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' && inputs.compare_greptimedb }} run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT/baseline" --data-root "$RUN_ROOT" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/runs/greptimedb" - - name: GreptimeDB load, correctness and benchmark - if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} + - name: Baseline GreptimeDB load, correctness and benchmark (S3) + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' && inputs.compare_greptimedb }} run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$BASELINE_GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ + --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ + --root "$RUN_ROOT/baseline/s3" --data-root "$RUN_ROOT" --owner "$OWNER-base-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/baseline/s3/runs/greptimedb" + - name: GreptimeDB load, correctness and benchmark (File) + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 's3' }} + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage file --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \ --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/runs/greptimedb" + - name: GreptimeDB load, correctness and benchmark (S3) + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} + if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' && inputs.greptimedb_storage != 'file' }} + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage target --greptimedb-storage s3 --target greptimedb --port 14000 --greptimedb-image "$GREPTIMEDB_IMAGE" \ + --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --query-timeout-seconds "$QUERY_TIMEOUT_SECONDS" --load-workers "$LOAD_WORKERS" \ + --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ + --root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + python3 o11ybench/scripts/check_tracesbench_query_plans.py "$RUN_ROOT/s3/runs/greptimedb" - name: VictoriaTraces load, correctness and benchmark if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victoriatraces == 'true' }} run: | @@ -422,49 +458,80 @@ jobs: - name: GreptimeDB version comparison if: always() && inputs.compare_greptimedb run: | - python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" >> "$GITHUB_STEP_SUMMARY" - - name: Validate and render report + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind traces --root "$RUN_ROOT" | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Validate and render primary report + if: always() && steps.generate.outcome == 'success' env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} run: | + if [[ "$GREPTIMEDB_STORAGE" == s3 ]]; then + TARGETS=$(tr "," "\n" <<< "$TARGETS" | sed "/^greptimedb$/d" | paste -sd, -) + [[ -n "$TARGETS" ]] || exit 0 + fi comparison_args=() - if [[ "$COMPARE_GREPTIMEDB" == true ]]; then + if [[ "$COMPARE_GREPTIMEDB" == true && "$GREPTIMEDB_STORAGE" != s3 ]]; then comparison_args+=(--baseline-run "$RUN_ROOT/baseline/runs/greptimedb/run.json") fi - python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "$TARGETS" "${comparison_args[@]}" \ + python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets "${TARGETS}" "${comparison_args[@]}" \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Validate and render S3 report + if: always() && steps.generate.outcome == 'success' && inputs.greptimedb_storage != 'file' + run: | + python3 o11ybench/scripts/run_tracesbench_docker.py --stage report --targets greptimedb \ + --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ + --root "$RUN_ROOT/s3" --data-root "$RUN_ROOT" --owner "$OWNER-s3" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Clean up owned containers + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} if: always() && steps.runtime.outcome == 'success' run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ + python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ --dataset "$DATASET" --runtime-cpus "$RUNTIME_CPUS" --runtime-memory "$RUNTIME_MEMORY" \ --root "$RUN_ROOT" --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Clean up baseline containers + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} if: always() && steps.runtime.outcome == 'success' && inputs.compare_greptimedb run: | - python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ + python3 greptimedb/.github/scripts/o11ybench-storage.py run -- python3 o11ybench/scripts/run_tracesbench_docker.py --stage cleanup \ --root "$RUN_ROOT/baseline" --owner "$OWNER-base" --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Benchmark summary if: always() - run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" + run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section benchmark | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" - name: Load / lifecycle summary if: always() - run: python3 greptimedb/.github/scripts/tracesbench-summary.py --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" + run: python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces --root "$RUN_ROOT" --targets "$TARGETS" --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" - name: Baseline load / lifecycle summary if: always() && inputs.compare_greptimedb run: | - echo '## Baseline GreptimeDB' >> "$GITHUB_STEP_SUMMARY" - python3 greptimedb/.github/scripts/tracesbench-summary.py \ - --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle >> "$GITHUB_STEP_SUMMARY" + echo '## Baseline GreptimeDB' | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py summary --kind traces \ + --root "$RUN_ROOT/baseline" --targets greptimedb --section lifecycle | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Render storage comparison overview + if: always() && steps.generate.outcome == 'success' + run: | + mkdir -p "$RUN_ROOT/metadata" + cp "$METADATA_ROOT"/*-image.json "$RUN_ROOT/metadata/" + python3 o11ybench/scripts/build_query_performance_report.py --traces "$RUN_ROOT" \ + --output "$RUN_ROOT-query-report.html" --json-output "$RUN_ROOT-query-report.json" + python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$RUN_ROOT-query-report.json" \ + | tee -a "$RUN_ROOT/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$RUN_ROOT" \ + --report "$RUN_ROOT-query-report.json" --targets "$TARGETS" - name: Stage benchmark artifacts if: always() run: | mkdir -p "$RUNNER_TEMP/tracesbench-artifacts" + cp "$RUN_ROOT"-query-report.* "$RUNNER_TEMP/tracesbench-artifacts/" 2>/dev/null || true if [[ -d "$METADATA_ROOT" ]]; then rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/tracesbench-artifacts/" fi - for entry in runs report report.json report.md version-comparison.json version-comparison.md; do + for entry in artifact-audit.json metadata runs report report.json report.md ci-summary.md version-comparison.json version-comparison.md; do if [[ -e "$RUN_ROOT/$entry" ]]; then rsync -a --no-owner --no-group "$RUN_ROOT/$entry" "$RUNNER_TEMP/tracesbench-artifacts/" fi @@ -473,6 +540,12 @@ jobs: mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/baseline" rsync -a --no-owner --no-group "$RUN_ROOT/baseline/runs" "$RUNNER_TEMP/tracesbench-artifacts/baseline/" fi + for arm in s3 baseline/s3; do + if [[ -d "$RUN_ROOT/$arm" ]]; then + mkdir -p "$RUNNER_TEMP/tracesbench-artifacts/$arm" + rsync -a --no-owner --no-group --exclude scratch --exclude corpora "$RUN_ROOT/$arm/" "$RUNNER_TEMP/tracesbench-artifacts/$arm/" + fi + done - name: Upload HTML and results (no corpus) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/vmbench-long-range.yml b/.github/workflows/vmbench-long-range.yml index 47daafed9e2..cf85f6630bf 100644 --- a/.github/workflows/vmbench-long-range.yml +++ b/.github/workflows/vmbench-long-range.yml @@ -2,6 +2,11 @@ name: VMBench Long Range on: workflow_dispatch: inputs: + greptimedb_storage: + description: 'GreptimeDB data storage (S3 requires AWS in the bucket region; WAL stays local)' + type: choice + options: [file, s3, both] + default: file run_greptimedb: description: Run GreptimeDB type: boolean @@ -41,7 +46,7 @@ on: default: v1.114.0@sha256:227ad53fd57a4b4430624898a1be0cac39431e1235fa0dc92f567cce38969344 required: true runtime_image: - description: 'Runtime image: auto = AWS ECR Public or Aliyun ACR by provider; explicit tag/digest overrides' + description: 'Runtime image: auto = pinned S3 test runtime on ACR; explicit tag/digest overrides' type: string default: auto required: true @@ -89,13 +94,18 @@ on: permissions: contents: read env: + GREPTIMEDB_STORAGE: ${{ inputs.greptimedb_storage || 'file' }} + S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }} + S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }} + RUNNER_REGION: ${{ vars.EC2_RUNNER_REGION }} + RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} COMPARE_GREPTIMEDB: ${{ inputs.compare_greptimedb }} BASELINE_GREPTIMEDB_TAG: ${{ inputs.baseline_greptimedb_tag }} BENCHMARK_ROOT: ${{ inputs.benchmark_root }} RANGE_PRESET: ${{ inputs.range_preset }} GREPTIMEDB_TAG: ${{ inputs.greptimedb_tag }} VICTORIAMETRICS_TAG: ${{ inputs.victoriametrics_tag }} - RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || inputs.provider == 'AWS' && 'public.ecr.aws/i8k6a5e1/greptime/o11ybenchruntime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:b43785b56f42@sha256:e38811eb68caf3682ba56f5591a3e8e27e78e7983a923abb29fa2d7cd8fdbd46' }} + RUNTIME_IMAGE: ${{ inputs.runtime_image != 'auto' && inputs.runtime_image || 'greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:s3-test-b0cf5204d2fc@sha256:c2bf622db6056a2029842f4dc1cc1f37b82773bd45b134a0a83ffe2a8cd62214' }} PROVIDER: ${{ inputs.provider }} REQUESTED_INSTANCE_TYPE: ${{ inputs.instance_type }} SYSTEM_DISK_GIB: ${{ inputs.system_disk_gib }} @@ -119,6 +129,10 @@ jobs: with: path: greptimedb persist-credentials: false + - name: Validate GreptimeDB storage + run: | + python3 greptimedb/tests/perf/test_o11ybench_storage.py + python3 greptimedb/.github/scripts/o11ybench-storage.py validate - name: Validate version comparison env: RUN_GREPTIMEDB: ${{ inputs.run_greptimedb }} @@ -173,7 +187,7 @@ jobs: [[ "$revision" =~ ^[0-9a-f]{40}$ ]] printf 'runtime_image=%s\no11ybench_ref=%s\nsystem_disk_gib=%s\n' "$resolved" "$revision" "$SYSTEM_DISK_GIB" >> "$GITHUB_OUTPUT" printf 'run_id=long-range-%s-%s\nplan_artifact=vmbench-plan-%s-%s\n' "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" "$GITHUB_RUN_ID" "$GITHUB_RUN_ATTEMPT" >> "$GITHUB_OUTPUT" - args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" + args=(--workload vmbench-long-range --range-preset "$RANGE_PRESET" --greptimedb-storage "$GREPTIMEDB_STORAGE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" --run-classification controlled_server_benchmark --no-open-browser) if [[ "$COMPARE_GREPTIMEDB" == true ]]; then @@ -192,10 +206,11 @@ jobs: # The packaged planner owns profile, target and resource-limit validation. mkdir -p "$RUNNER_TEMP/vmbench-plans" # Keep the original warm defaults: serial 5/1 and c1/c4 30s/5s. - docker run --rm "$resolved" scripts/o11ybench_plan.py "${args[@]}" \ + docker run --rm -e S3_BUCKET -e S3_REGION "$resolved" scripts/o11ybench_plan.py "${args[@]}" \ --run-id "long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \ --output-root "$BENCHMARK_ROOT/o11ybench/o11ybench-long-range-$GITHUB_RUN_ID-$GITHUB_RUN_ATTEMPT-warm" \ --cache-policy warm > "$RUNNER_TEMP/vmbench-plans/warm.json" + python3 greptimedb/.github/scripts/o11ybench-storage.py normalize-metrics "$RUNNER_TEMP/vmbench-plans/warm.json" - uses: actions/upload-artifact@v4 with: name: vmbench-plan-${{ github.run_id }}-${{ github.run_attempt }} @@ -340,25 +355,87 @@ jobs: --plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --dry-run \ > "$METADATA_ROOT/warm-execution-plan.json" cat "$METADATA_ROOT/warm-execution-plan.json" - - name: Warm serial and concurrent benchmarks + - name: Generate shared dataset and split target plans + id: generate run: | - set -euo pipefail - python3 -u o11ybench/scripts/run_benchmark_plan.py \ - --plan "$METADATA_ROOT/warm.json" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --execute \ - 2>&1 | tee "$METADATA_ROOT/warm-execution.log" - python3 o11ybench/scripts/check_run_benchmark_plan.py --artifacts "$ARTIFACT_ROOT-warm" + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-prepare \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Load and benchmark Baseline GreptimeDB File + if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 's3' }} + run: | + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-baseline \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Load and benchmark Baseline GreptimeDB S3 + if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.compare_greptimedb && inputs.greptimedb_storage != 'file' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3-baseline \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Load and benchmark GreptimeDB File + if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 's3' }} + run: | + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Load and benchmark GreptimeDB S3 + if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_greptimedb && inputs.greptimedb_storage != 'file' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target greptimedb-s3 \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Load and benchmark VictoriaMetrics + if: ${{ !cancelled() && steps.generate.outcome == 'success' && inputs.run_victoriametrics }} + run: | + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-run --target victoriametrics \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" + - name: Validate all selected target results + if: always() && steps.generate.outcome == 'success' + run: | + python3 greptimedb/.github/scripts/o11ybench-storage.py metrics-validate \ + --plan "$METADATA_ROOT/warm.json" --tools o11ybench --runtime-image "$RESOLVED_RUNTIME_IMAGE" - name: Clean up owned containers + env: + AWS_ACCESS_KEY_ID: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_ACCESS_KEY_ID || '' }} + AWS_SECRET_ACCESS_KEY: ${{ inputs.greptimedb_storage != 'file' && secrets.AWS_SECRET_ACCESS_KEY || '' }} if: always() run: | docker ps -aq --filter "label=o11ybench.run-id=$RUN_ID-warm" | xargs -r docker rm -fv + for name in greptimedb greptimedb-s3 greptimedb-baseline greptimedb-s3-baseline victoriametrics; do + plan="$METADATA_ROOT/$name.json" + [[ -f "$plan" ]] || continue + run_id=$(jq -r '.run_id // empty' "$plan") + [[ -n "$run_id" ]] || continue + docker ps -aq --filter "label=o11ybench.run-id=$run_id" | xargs -r docker rm -fv + done + if [[ -d "$ARTIFACT_ROOT-warm" ]]; then + for probe in "$ARTIFACT_ROOT-warm"/o11ybench/*/*/s3-probe.json; do + [[ -f "$probe" ]] || continue + python3 o11ybench/scripts/greptimedb_storage.py recover --runtime-image "$RESOLVED_RUNTIME_IMAGE" --output "${probe%/*}" + done + fi - name: GreptimeDB version comparison if: always() && inputs.compare_greptimedb run: | - python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/benchmark-version-comparison.py report --kind long-range --root "$ARTIFACT_ROOT-warm" | tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + - name: Render storage comparison overview + if: always() && steps.generate.outcome == 'success' + run: | + mkdir -p "$ARTIFACT_ROOT-warm/metadata" + cp "$METADATA_ROOT"/*-image.json "$ARTIFACT_ROOT-warm/metadata/" + python3 o11ybench/scripts/build_query_performance_report.py --metrics "$ARTIFACT_ROOT-warm" \ + --output "$ARTIFACT_ROOT-warm-query-report.html" --json-output "$ARTIFACT_ROOT-warm-query-report.json" + python3 greptimedb/.github/scripts/o11ybench-storage.py overview "$ARTIFACT_ROOT-warm-query-report.json" \ + | tee -a "$ARTIFACT_ROOT-warm/ci-summary.md" >> "$GITHUB_STEP_SUMMARY" + python3 greptimedb/.github/scripts/o11ybench-storage.py audit --root "$ARTIFACT_ROOT-warm" \ + --report "$ARTIFACT_ROOT-warm-query-report.json" --targets "$(jq -r '[.targets[].kind] | unique | join(",")' "$METADATA_ROOT/warm.json")" - name: Stage benchmark artifacts if: always() run: | mkdir -p "$RUNNER_TEMP/vmbench-artifacts" + cp "$ARTIFACT_ROOT-warm"-query-report.* "$RUNNER_TEMP/vmbench-artifacts/" 2>/dev/null || true if [[ -d "$METADATA_ROOT" ]]; then rsync -a --no-owner --no-group "$METADATA_ROOT" "$RUNNER_TEMP/vmbench-artifacts/" fi diff --git a/tests/perf/test_ci_runner_scripts.py b/tests/perf/test_ci_runner_scripts.py index e24cf5f3e71..b1c8f118bfb 100644 --- a/tests/perf/test_ci_runner_scripts.py +++ b/tests/perf/test_ci_runner_scripts.py @@ -625,7 +625,7 @@ class BenchmarkImagePullTest(unittest.TestCase): for reference in ("RUNTIME_IMAGE", "RESOLVED_RUNTIME_IMAGE", "reference"): self.assertIn(f'pull-benchmark-image.sh "${reference}"', long_range) self.assertLess(long_range.index('pull-benchmark-image.sh "$reference"'), - long_range.index('- name: Warm serial and concurrent benchmarks')) + long_range.index('- name: Generate shared dataset and split target plans')) self.assertIn('pull-benchmark-image.sh "$reference"', agent) self.assertLess(agent.index('resolve_image runtime'), agent.index('- name: Generate dataset once')) self.assertIn('pull-benchmark-image.sh "$RUNTIME_IMAGE"', traces) diff --git a/tests/perf/test_o11ybench_storage.py b/tests/perf/test_o11ybench_storage.py new file mode 100644 index 00000000000..308469fd165 --- /dev/null +++ b/tests/perf/test_o11ybench_storage.py @@ -0,0 +1,111 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Offline contracts for File/S3 benchmark arm selection.""" +import importlib.util +import copy +import json +import tempfile +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('storage', ROOT / '.github/scripts/o11ybench-storage.py') +storage = importlib.util.module_from_spec(spec) +spec.loader.exec_module(storage) + + +class StorageTests(unittest.TestCase): + def test_admission(self): + self.assertEqual(storage.validate({}), 'file') + env = dict(GREPTIMEDB_STORAGE='both', RUN_GREPTIMEDB='true', PROVIDER='AWS', + S3_BUCKET='test', S3_REGION='us-east-1', RUNNER_REGION='us-east-1') + self.assertEqual(storage.validate(env), 'both') + for key, value in [('PROVIDER', 'Aliyun'), ('RUNNER_REGION', 'us-west-2'), + ('S3_BUCKET', ''), ('RUN_GREPTIMEDB', 'false'), ('GREPTIMEDB_STORAGE', 'invalid')]: + with self.assertRaises(ValueError): + storage.validate(dict(env, **{key: value})) + def test_execution_must_not_expand_arms(self): + command = ['runner', '--stage', 'target', '--root', '/run', '--owner', 'owner'] + with self.assertRaises(ValueError): + storage.commands(command, 'both') + + def test_metrics_split_preserves_data_and_target_contract(self): + plan = dict(output_root='/tmp/o11ybench-test', run_id='test', + data=dict(generate=True, data_home='/tmp/o11ybench-test/data'), + targets=[dict(name='greptimedb', storage=dict(type='file')), + dict(name='greptimedb-s3', storage=dict(type='s3', root='owned/prefix/'))], + report=dict(root='old', index_html='old/index.html'), + artifacts=dict(run_manifest='old/run-manifest.json'), + query=dict(modes=['serial'], serial_tries=5)) + original = copy.deepcopy(plan) + children = storage.split_metrics(plan) + self.assertEqual(plan, original) + for name, child in children.items(): + self.assertEqual(child['query'], plan['query']) + self.assertEqual(child['data'], dict(plan['data'], generate=False)) + self.assertEqual(len(child['targets']), 1) + self.assertEqual(child['targets'][0]['name'], name) + self.assertEqual(child['output_root'], '/tmp/o11ybench-test/o11ybench/o11ybench-' + name) + self.assertTrue(child['report']['root'].startswith(child['output_root'])) + self.assertNotEqual(children['greptimedb']['run_id'], children['greptimedb-s3']['run_id']) + + def test_missing_s3_arm_is_not_a_success(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + report = root / 'report.json' + report.write_text(json.dumps(dict(workloads=[dict(participants=[ + dict(target='greptimedb', role='candidate', storage=dict(type='file'))], scenarios=[])]))) + with self.assertRaisesRegex(ValueError, 'participant mismatch'): + storage.audit(root, report, ['greptimedb'], 'both') + + def test_s3_summary_is_explicit_when_missing(self): + with tempfile.TemporaryDirectory() as tmp: + text = storage.summary('agent', Path(tmp), ['greptimedb'], 'benchmark', 'both') + self.assertIn('GreptimeDB File', text) + self.assertIn('GreptimeDB S3', text) + self.assertIn('S3 cleanup: **failed / unavailable**', text) + + def test_cleanup_both(self): + command = ['runner', '--stage', 'cleanup', '--root', '/run', '--owner', 'owner'] + calls = storage.commands(command, 'both') + self.assertEqual(len(calls), 2) + self.assertTrue(all('--greptimedb-storage' not in c for c in calls)) + def test_cleanup_s3_only_still_visits_primary_competitors(self): + command = ['runner', '--stage', 'cleanup', '--root', '/run', '--owner', 'owner'] + calls = storage.commands(command, 's3') + self.assertEqual(len(calls), 2) + self.assertEqual(calls[1][calls[1].index('--root') + 1], '/run/s3') + + def test_workflow_wiring(self): + for name in ('agent-observability', 'tracesbench', 'vmbench-long-range'): + text = (ROOT / '.github/workflows' / (name + '.yml')).read_text() + self.assertIn('options: [file, s3, both]', text) + self.assertIn('o11ybench-storage.py validate', text) + self.assertIn('o11ybench-storage.py audit', text) + self.assertIn('ci-summary.md', text) + self.assertIn('AWS_SECRET_ACCESS_KEY:', text) + self.assertNotIn('o11ybench-storage.py run -- bash o11ybench/scripts/run-agent-observability.sh --stage target', text) + if name == 'vmbench-long-range': + self.assertIn('name: Load and benchmark GreptimeDB File', text) + self.assertIn('name: Load and benchmark GreptimeDB S3', text) + self.assertIn('metrics-prepare', text) + self.assertIn('metrics-validate', text) + else: + self.assertIn('--stage target --greptimedb-storage file', text) + self.assertIn('--stage target --greptimedb-storage s3', text) + + +if __name__ == '__main__': + unittest.main()