diff --git a/.github/scripts/agent-observability-summary.py b/.github/scripts/agent-observability-summary.py new file mode 100644 index 0000000000..38ba4e9dbd --- /dev/null +++ b/.github/scripts/agent-observability-summary.py @@ -0,0 +1,220 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import html +import json +import math +from pathlib import Path + +TARGETS = ('greptimedb', 'clickhouse', 'victorialogs') + + +def duration(ms): + if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0: + return '—' + if ms < 1: + return f'{ms * 1000:.0f} µs' + if ms < 1000: + return f'{ms:.1f} ms' + if ms < 60_000: + return f'{ms / 1000:.2f} s' + minutes, seconds = divmod(round(ms / 1000), 60) + hours, minutes = divmod(minutes, 60) + return f'{hours}h {minutes}m {seconds}s' if hours else f'{minutes}m {seconds}s' + + +def size(value): + if not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + return '—' + for unit in ('B', 'KiB', 'MiB', 'GiB', 'TiB'): + if value < 1024 or unit == 'TiB': + return f'{value:.2f} {unit}' + value /= 1024 + + +def category(qid): + if qid in ('Q01', 'Q03', 'Q04'): + return 'Scan / index' + if qid in ('Q05', 'Q08', 'Q11', 'Q14', 'Q15'): + return 'Keyword search' + if qid in ('Q02', 'Q06', 'Q09', 'Q10', 'Q13', 'Q18', 'Q19', 'Q20'): + return 'Aggregation' + return '—' + + +def cell(value): + return html.escape(str(value if value is not None else '—')).replace('|', '|').replace('\n', ' ') + + +def read(path): + if not path.exists(): + return {} + return json.loads(path.read_text()) + + +def table(headers, rows): + return '\n'.join(['| ' + ' | '.join(map(cell, headers)) + ' |', + '| ' + ' | '.join('---' for _ in headers) + ' |'] + + ['| ' + ' | '.join(map(cell, row)) + ' |' for row in rows]) + + +def delta(value, baseline): + if not all(isinstance(v, (int, float)) and math.isfinite(v) for v in (value, baseline)): + return '—' + return f'{(value / baseline - 1) * 100:+.1f}%' if baseline > 0 and value >= 0 else '—' + + +def result_dir(root, target): + return root / f'{target}-1' / f'{target}-1' + + +def benchmark(root, targets): + summaries = {t: read(result_dir(root, t) / 'measured-summary.json') for t in targets} + base = summaries.get('greptimedb', {}) + base_queries = {q['query_id']: q for q in base.get('per_query', [])} + ids = sorted({q['query_id'] for s in summaries.values() for q in s.get('per_query', [])}) + lines = ['# Benchmark summary', '', + 'Latency Δ = (target / GreptimeDB − 1) × 100%; positive is slower, negative is faster. ' + 'Missing, failed or incompatible results have no comparison. Counts exclude warmup.', '', + 'Percentiles from small smoke samples are diagnostic, not stable performance estimates.', ''] + for t, s in summaries.items(): + lines.append(f'- **{t}**: {"passed" if s.get("passed") is True else "failed / unavailable"}; ' + f'warmup/query: {cell(s.get("warmup_runs"))}; measured/query: {cell(s.get("measured_runs"))}.') + rows = [] + for qid in ids: + for t, s in summaries.items(): + q = next((q for q in s.get('per_query', []) if q['query_id'] == qid), {}) + compatible = (t != 'greptimedb' and s.get('passed') is True and base.get('passed') is True + and q.get('result_fingerprint') is not None + and q.get('result_fingerprint') == base_queries.get(qid, {}).get('result_fingerprint') + and all(s.get(k) == base.get(k) and s.get(k) is not None + for k in ('contract_sha256', 'data_profile', 'query_window'))) + row = [qid, category(qid), t] + for key in ('p50_ms', 'p95_ms', 'p99_ms'): + row += [duration(q.get(key)), delta(q.get(key), base_queries.get(qid, {}).get(key)) if compatible else '—'] + rows.append(row + [q.get('success'), q.get('errors'), q.get('timeouts')]) + lines += ['', table(['Query', 'Category', 'DB', 'P50', 'Δ', 'P95', 'Δ', 'P99', 'Δ', 'Success', 'Errors', 'Timeouts'], rows)] + comparison = read(root / 'comparison.json') + lines += ['', 'Cross-target comparison: ' + ('passed' if comparison.get('passed') is True else + 'failed' if comparison else 'not available / not run'), ''] + lines += ['
', 'EXPLAIN diagnostics (after measurement)', ''] + for t in targets: + plans = read(result_dir(root, t) / 'query-plans.json') + if not plans: + lines += [f'**{t}**: unavailable.', ''] + continue + capability = plans.get('capability', {}) + if capability.get('state') == 'unsupported': + lines += [f'**{t}**: {cell(capability.get("reason"))}', ''] + continue + for q in plans.get('queries', []): + # Bound summary size; complete responses remain in the uploaded JSON. + raw = str(q.get('raw_response') or q.get('error') or '') + limit = 3000 + excerpt = raw[:limit] + ('\n[Truncated; see query-plans.json artifact.]' if len(raw) > limit else '') + lines += ['
', f'{cell(t)} / {cell(q.get("query_name"))}: {cell(q.get("state"))}', '', + '
' + html.escape(str(q.get('explain_sql', ''))[:2000]) + '
', + '
' + html.escape(excerpt) + '
', '', '
', ''] + lines += ['
', ''] + return '\n'.join(lines) + + +def lifecycle(root, targets): + manifest = read(root / 'run-manifest.json') + corpus = read(root / 'corpus' / 'summary.json') + lines = ['# Load / lifecycle summary', '', + f'Dataset: **{cell(manifest.get("profile", corpus.get("data_profile", {}).get("id")))}**; rows: **{cell(corpus.get("line_count"))}**; ' + f'seed: `{cell(corpus.get("seed"))}`.', + f'DB limits: {cell(manifest.get("db_cpus"))} CPU / {cell(manifest.get("db_memory"))}.', + f'Data window: {cell(corpus.get("start"))} → {cell(corpus.get("end"))}.', + f'Query window: {cell(corpus.get("query_start"))} → {cell(corpus.get("query_end"))}.', ''] + rows = [] + for t in targets: + d = result_dir(root, t) + load = read(d / 'load-result.json') + check = read(d / 'correctness-result.json') + state_path = root / f'{t}-1' / 'target-after-load.json' + states = read(state_path) + state = states[0].get('State', {}) if states else {} + exit_path = root / f'{t}-1' / 'exit-code.txt' + rows.append([t, load.get('count_before'), load.get('count_after'), load.get('smoke_matched_rows'), + sum(q.get('status') == 'passed' for q in check.get('queries', [])) if check else None, + state.get('Running'), state.get('OOMKilled'), + exit_path.read_text().strip() if exit_path.exists() else None]) + lines += [table(['DB', 'Rows before', 'Rows after', 'Load smoke rows', 'Correct queries', + 'Running after load', 'OOM after load', 'Target exit code'], rows), '', + ] + physical_rows = [] + load_rows = [] + details = ['
', 'DDL, index configuration and image provenance', ''] + raw_bytes = read(root / 'corpus' / 'size.json').get('raw_jsonl_bytes') + for t in targets: + directory = result_dir(root, t) + physical = read(directory / 'physical-evidence.json') + storage = physical.get('storage', {}) + total = storage.get('engine_total_bytes') + stats = read(root / f'{t}-1' / 'post-load-docker-stats.json') + ratio = f'{raw_bytes / total:.2f}×' if (physical.get('passed') is True and + isinstance(raw_bytes, (int, float)) and raw_bytes > 0 and + isinstance(total, (int, float)) and total > 0) else '—' + physical_rows.append([t, size(total), ratio, physical.get('materialization', {}).get('state'), + storage.get('sst_files', storage.get('active_parts')), + stats.get('CPUPerc'), stats.get('MemUsage')]) + load = read(directory / 'load-result.json') + metadata = read(directory / 'target-metadata.json') + load_rows.append([t, load.get('load_transport', load.get('transport', '/insert/jsonline' if t == 'victorialogs' and load else None)), + load.get('batches', load.get('batch_count')), load.get('smoke_matched_rows')]) + ddl = metadata.get('schema_sql') or physical.get('schema_definition') + detail = {'image': manifest.get('images', {}).get(t), 'schema': ddl, + 'physical_design': metadata.get('physical_design'), + 'storage': storage, 'coverage': physical.get('coverage')} + details += [f'### {t}', '
' + html.escape(json.dumps(detail, indent=2, ensure_ascii=False)) + '
', ''] + lines += ['## Load and physical evidence', '', table(['DB', 'Write transport', 'Batches', 'Load smoke rows'], load_rows), '', + f'Raw JSONL size: **{size(raw_bytes)}**. Ratio = raw JSONL / engine bytes at collection time; ' + 'not a settled-storage guarantee. CPU/memory are post-load snapshots, not peaks. ' + 'Missing artifacts remain blank; pre-cleanup stats are not substituted.', '', + table(['DB', 'Engine size', 'Size ratio', 'Materialization', 'SSTs / active parts', + 'Post-load CPU', 'Post-load memory / limit'], physical_rows), '', *details, '
', ''] + rows = [] + for context, path in [('dataset / final cleanup', root / 'timings.jsonl')] + [ + (t, root / f'{t}-1' / 'timings.jsonl') for t in targets]: + if not path.exists(): + rows.append([context, 'unavailable', '—', '—']) + continue + for line in path.read_text().splitlines(): + entry = json.loads(line) + # Older date implementations emitted ns-like values under ms keys; never guess units. + valid = all(isinstance(entry.get(k), (int, float)) and 0 <= entry[k] < 100_000_000_000_000 + for k in ('started_at_ms', 'ended_at_ms')) + rows.append([context, entry.get('phase'), duration(entry.get('elapsed_ms')) if valid else + 'invalid timestamp units (legacy artifact)', entry.get('exit_code')]) + lines += ['## Phase timings', '', 'Phase duration includes orchestration overhead. Container cleanup does not prove ECS teardown.', '', table(['DB / scope', 'Phase', 'Duration', 'Exit code'], rows), ''] + return '\n'.join(lines) + + +def main(): + parser = argparse.ArgumentParser(description='Render observability artifacts as GitHub step summaries.') + parser.add_argument('--root', type=Path, required=True) + parser.add_argument('--targets', required=True) + parser.add_argument('--section', choices=('benchmark', 'lifecycle'), required=True) + args = parser.parse_args() + targets = args.targets.split(',') + if not targets or any(t not in TARGETS for t in targets) or len(set(targets)) != len(targets): + parser.error('targets must be a comma-separated subset of greptimedb,clickhouse,victorialogs') + print((benchmark if args.section == 'benchmark' else lifecycle)(args.root, targets)) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/agent-observability.yml b/.github/workflows/agent-observability.yml index f1507d141e..02c4ccb81b 100644 --- a/.github/workflows/agent-observability.yml +++ b/.github/workflows/agent-observability.yml @@ -48,7 +48,7 @@ on: runtime_image: description: Aliyun runtime image (tag or digest reference) type: string - default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-39fadf81@sha256:3390c7810a14a926870cb132ab97fef4dce3b4dc801e15b3bf8a870bd26eb035 + default: greptime-registry.cn-hangzhou.cr.aliyuncs.com/tools/o11ybench-runtime:20260917-8111034e@sha256:ce9dee2abb1e70f907a09ea01854904a91cc97748b0c4ef6f33d9ec34044bec0 required: true ecs_instance_type: description: ECS instance type (independent of Query Regression) @@ -111,7 +111,9 @@ jobs: path: greptimedb persist-credentials: false - name: Test ECS lifecycle helpers - run: python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py + run: | + python3 greptimedb/tests/perf/test_aliyun_ecs_runner_scripts.py + python3 greptimedb/tests/perf/test_agent_observability_summary.py - name: Validate requested benchmark id: inputs shell: bash @@ -257,6 +259,7 @@ jobs: uses: actions/checkout@v4 with: repository: GreptimeTeam/o11ybench + token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} ref: ${{ env.RESOLVED_O11YBENCH_REF }} path: o11ybench persist-credentials: false @@ -266,7 +269,7 @@ jobs: - name: Generate dataset once id: generate run: |- - bash o11ybench/scripts/run-agent-observability.sh --stage generate --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage generate --execute \ --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" \ --root "$ARTIFACT_ROOT" --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" cp "$MANIFEST_PATH" "$ARTIFACT_ROOT/run-manifest.json" @@ -274,7 +277,7 @@ jobs: id: greptimedb if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.greptimedb == 'true' }} run: |- - bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ --target greptimedb --image "$GREPTIMEDB_IMAGE" \ @@ -283,7 +286,7 @@ jobs: id: clickhouse if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.clickhouse == 'true' }} run: |- - bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ --target clickhouse --image "$CLICKHOUSE_IMAGE" \ @@ -292,7 +295,7 @@ jobs: id: victorialogs if: ${{ !cancelled() && steps.generate.outcome == 'success' && needs.validate.outputs.victorialogs == 'true' }} run: |- - bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage target --execute \ --owner "$OWNER" --runtime-image "$RESOLVED_RUNTIME_IMAGE" --root "$ARTIFACT_ROOT" \ --profile "$PROFILE" --db-cpus "$DB_CPUS" --db-memory "$DB_MEMORY" \ --target victorialogs --image "$VICTORIALOGS_IMAGE" \ @@ -312,9 +315,19 @@ jobs: if: always() run: |- if [[ -f o11ybench/scripts/run-agent-observability.sh ]]; then - bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ + GITHUB_STEP_SUMMARY= bash o11ybench/scripts/run-agent-observability.sh --stage cleanup --execute \ --owner "$OWNER" --root "$ARTIFACT_ROOT" fi + - name: Benchmark summary + if: always() + run: | + python3 greptimedb/.github/scripts/agent-observability-summary.py \ + --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section benchmark >> "$GITHUB_STEP_SUMMARY" + - name: Load / lifecycle summary + if: always() + run: | + python3 greptimedb/.github/scripts/agent-observability-summary.py \ + --root "$ARTIFACT_ROOT" --targets "$TARGETS" --section lifecycle >> "$GITHUB_STEP_SUMMARY" - name: Upload results (no dataset) if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 682a2e5921..9615166256 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -89,6 +89,7 @@ jobs: python3 tests/perf/test_query_regression_nightly_refs.py python3 tests/perf/test_query_regression_slash.py python3 tests/perf/test_aliyun_ecs_runner_scripts.py + python3 tests/perf/test_agent_observability_summary.py check: if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} diff --git a/tests/perf/test_agent_observability_summary.py b/tests/perf/test_agent_observability_summary.py new file mode 100644 index 0000000000..8223231c90 --- /dev/null +++ b/tests/perf/test_agent_observability_summary.py @@ -0,0 +1,124 @@ +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).parents[2] / '.github/scripts/agent-observability-summary.py' +spec = importlib.util.spec_from_file_location('observability_summary', SCRIPT) +summary = importlib.util.module_from_spec(spec) +spec.loader.exec_module(summary) + + +class SummaryTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + + def write(self, path, data): + path = self.root / path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + return path + + def measured(self, target, value=10, **overrides): + data = dict(passed=True, contract_sha256='same', data_profile={'id':'S'}, + query_window={'start':0,'end':1}, warmup_runs=1, measured_runs=5, + per_query=[dict(query_id='Q01', p50_ms=value, p95_ms=value, + p99_ms=value, result_fingerprint='same-result', success=5, errors=0, timeouts=0)]) + data.update(overrides) + return self.write(f'{target}-1/{target}-1/measured-summary.json', data) + + def test_duration_units_and_invalid(self): + for value, expected in [(0.25,'250 µs'), (12.3,'12.3 ms'), (1250,'1.25 s'), + (130000,'2m 10s'), (3661000,'1h 1m 1s'), + (None,'—'), (-1,'—'), (float('nan'),'—')]: + with self.subTest(value=value): + self.assertEqual(summary.duration(value), expected) + + def test_relative_latency_and_measurement_counts(self): + self.measured('greptimedb') + self.measured('clickhouse',20) + self.measured('victorialogs',5) + text = summary.benchmark(self.root, summary.TARGETS) + self.assertIn('| Q01 | Scan / index | greptimedb | 10.0 ms | — |', text) + self.assertEqual(text.count('+100.0%'),3) + self.assertEqual(text.count('-50.0%'),3) + self.assertEqual(text.count('| 5 | 0 | 0 |'),3) + + def test_missing_failed_and_incompatible_baseline(self): + self.measured('clickhouse',20) + for base in [None, {'passed':False}, {'data_profile':{'id':'M'}}]: + if base is not None: + self.measured('greptimedb', **base) + text = summary.benchmark(self.root, summary.TARGETS) + self.assertNotIn('+100.0%',text) + self.assertEqual(summary.delta(10,0),'—') + self.assertEqual(summary.delta(None,10),'—') + + def test_mismatched_results_have_no_comparison(self): + self.measured('greptimedb') + path = self.measured('clickhouse', 20) + data = json.loads(path.read_text()) + data['per_query'][0]['result_fingerprint'] = 'different-result' + path.write_text(json.dumps(data)) + self.assertNotIn('+100.0%', summary.benchmark(self.root, summary.TARGETS)) + + def test_collapsed_explain_escaped_and_bounded(self): + self.write('greptimedb-1/greptimedb-1/query-plans.json', + {'capability':{'state':'supported'}, 'queries':[ + {'query_name':'Q01','state':'passed','explain_sql':'SELECT ', + 'raw_response':'