rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-13 07:22:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: fail a benchmark cell whose metrics scrape fails and zero untouched WAL counters

Browse Source 
A load that succeeds without its metrics scrape now fails the cell with exit
code 4 and records metrics_scraped in cell.json. The summary reports an
event counter that never registered as zero when the cell is an object store
WAL cell whose scrape succeeded and created objects, and as unknown otherwise.
Planned cells that never ran are read through the same path as recorded ones.

Signed-off-by: jeremyhi <fengjiachun@gmail.com>
This commit is contained in:
jeremyhi
2026-10-12 17:19:39 +08:00
parent 834877b10d
commit 364dc16f7a
5 changed files with 83 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/oswal-s3-bench-run.sh
+11 -17
View File
@@ -1,11 +1,5 @@
#!/usr/bin/env bash
# Host side of the object store WAL S3 benchmark (oswal-s3-bench.yml).
# check-runtime IMAGE resolve the o11ybench runtime and check it carries the loader and generator
# pull pull the runtime and GreptimeDB images and record their metadata
# wal-disk DIR mount the instance store at DIR when the host has one
# generate generate the dataset once with the o11ybench runtime
# cells run every cell of $PLAN, each on a fresh GreptimeDB container
# scrub refuse results that contain a credential value
# Host side of oswal-s3-bench.yml: one subcommand per step; each cell of $PLAN gets a fresh GreptimeDB.
set -euo pipefail
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -64,10 +58,10 @@ generate() {
# Scrape once, after the load succeeded or failed, while the server still runs.
scrape_metrics() {
if curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all"; then
grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true
fi
curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all" || { rm -f "$dir/metrics.all"; return 1; }
grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true
rm -f "$dir/metrics.all"
metrics_scraped=true
}
finish() {
@@ -75,15 +69,16 @@ finish() {
trap - EXIT
set +e
if [[ -n "$cid" ]]; then
scrape_metrics
# A load that succeeded without its metrics is not a measurement.
scrape_metrics || { echo "$id: metrics scrape failed" >&2; [[ $rc != 0 ]] || rc=4; }
docker stop -t 120 "$cid" >/dev/null
docker logs "$cid" > "$dir/server.log" 2>&1
docker rm -fv "$cid" >/dev/null
fi
[[ -z "$wal_dir" ]] || sudo -n rm -rf "$wal_dir"
jq -n --argjson spec "$spec" --arg device "$wal_device" --arg version "$version" \
--arg load_seconds "$load_seconds" --argjson exit_code "$rc" \
'$spec + {wal_device: $device, version: $version, exit_code: $exit_code,
--arg load_seconds "$load_seconds" --argjson exit_code "$rc" --argjson scraped "$metrics_scraped" \
'$spec + {wal_device: $device, version: $version, exit_code: $exit_code, metrics_scraped: $scraped,
load_wall_seconds: ($load_seconds | tonumber? // null)}' > "$dir/cell.json"
exit "$rc"
}
@@ -92,10 +87,9 @@ finish() {
# global because the EXIT trap runs after this function returns.
cell() {
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGE:?}"
: "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}"
: "${AWS_ACCESS_KEY_ID:?}" "${AWS_SECRET_ACCESS_KEY:?}"
spec=$1 cid= wal_dir= version= load_seconds=
local id concurrency name setting started ended
: "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}" "${AWS_ACCESS_KEY_ID:?}" "${AWS_SECRET_ACCESS_KEY:?}"
spec=$1 cid= wal_dir= version= load_seconds= metrics_scraped=false id=
local concurrency name setting started ended
id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec")
dir="$RESULTS/cells/$id"; name="$OWNER-$id"
mkdir -p "$dir"
.github/scripts/oswal-s3-bench-summary.py
+17 -20
View File
@@ -64,8 +64,10 @@ def geomean(values: list) -> float | None:
return math.exp(sum(map(math.log, values)) / len(values)) if values and all(v > 0 for v in values) else None
def load_cell(path: Path) -> dict:
cell = json.loads((path / "cell.json").read_text())
def load_cell(path: Path, spec: dict | None = None) -> dict:
"""One cell's row; a planned cell that never recorded a result has no values, so no ratio uses its round."""
recorded = path / "cell.json"
cell = json.loads(recorded.read_text()) if recorded.exists() else spec | {"exit_code": None, "missing": True}
result_path = path / "greptimedb-1" / "load-result.json"
result = json.loads(result_path.read_text()) if result_path.exists() else {}
metrics_path = path / "metrics.prom"
@@ -74,27 +76,24 @@ def load_cell(path: Path) -> dict:
load_seconds = result.get("load_seconds")
row = {
"id": cell["id"], "round": cell["round"], "concurrency": cell["concurrency"],
"config": cell["config"], "exit_code": cell["exit_code"], "wal_device": cell["wal_device"],
"config": cell["config"], "exit_code": cell["exit_code"], "wal_device": cell.get("wal_device"),
"missing": cell.get("missing", False),
"version": cell.get("version"), "load_wall_seconds": cell.get("load_wall_seconds"),
"rows": result.get("count_after"), "load_seconds": load_seconds,
"rows_per_second": result.get("rows_per_second") if cell["exit_code"] == 0 else None,
"append_ack_p50_seconds": quantile(0.5, ack), "append_ack_p99_seconds": quantile(0.99, ack),
}
# The counters register on their first event, so a scrape of an object store WAL
# that created objects omits the ones that never moved: those are zero, not unknown.
active = (cell.get("metrics_scraped") and cell.get("wal", {}).get("PROVIDER") == "experimental_object_store"
and metrics["samples"].get("created_objects_total"))
for counter in COUNTERS:
row[counter] = metrics["samples"].get(counter)
row[counter] = metrics["samples"].get(counter, 0 if active else None)
created = row["created_objects_total"]
row["creates_per_minute"] = created * 60 / load_seconds if created is not None and load_seconds else None
return row
def missing_cell(spec: dict) -> dict:
"""A planned cell that never recorded a result, so no ratio can use its round."""
row = {key: None for key in ("exit_code", "wal_device", "version", "load_wall_seconds", "rows",
"load_seconds", "rows_per_second", "append_ack_p50_seconds",
"append_ack_p99_seconds", "creates_per_minute", *COUNTERS)}
return row | {key: spec[key] for key in ("id", "round", "concurrency", "config")} | {"missing": True}
def ratios(rows: list) -> list:
"""D/R and E/R per round and their geometric mean over the rounds, per concurrency."""
speed = {(r["round"], r["concurrency"], r["config"]): r["rows_per_second"] for r in rows}
@@ -124,10 +123,8 @@ def render(run: dict, rows: list, pairs: list) -> str:
"| Stalled | Poisoned | Create failures | Conflicts | WAL on |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"]
for r in rows:
if r.get("missing"):
cell = f"{r['id']} (not run)"
else:
cell = r["id"] if r["exit_code"] == 0 else f"{r['id']} (failed, exit {r['exit_code']})"
cell = f"{r['id']} (not run)" if r["missing"] else r["id"] if r["exit_code"] == 0 \
else f"{r['id']} (failed, exit {r['exit_code']})"
lines.append(" | ".join([
f"| {cell}", fmt(r["rows_per_second"]), fmt(r["load_seconds"], 1), fmt(r["load_wall_seconds"], 1),
fmt(r["rows"]), fmt(r["append_ack_p50_seconds"], 1, 1000), fmt(r["append_ack_p99_seconds"], 1, 1000),
@@ -149,12 +146,12 @@ def main() -> None:
run = json.loads(run_path.read_text()) if run_path.exists() else {}
plan_path = args.root / "plan.json"
planned = json.loads(plan_path.read_text())["cells"] if plan_path.exists() else []
cells = {path.name: load_cell(path) for path in (args.root / "cells").glob("*") if (path / "cell.json").exists()}
# The plan, not what happened to run, defines the expected cells and rounds.
rows = [cells.get(spec["id"]) or missing_cell(spec) for spec in planned]
rows += [cells[i] for i in sorted(cells) if i not in {spec["id"] for spec in planned}]
rows = [load_cell(args.root / "cells" / spec["id"], spec) for spec in planned]
rows += [load_cell(path) for path in sorted((args.root / "cells").glob("*"))
if path.name not in {spec["id"] for spec in planned} and (path / "cell.json").exists()]
pairs = ratios(rows)
missing = [r["id"] for r in rows if r.get("missing")]
missing = [r["id"] for r in rows if r["missing"]]
(args.root / "summary.json").write_text(json.dumps(
{"run": run, "planned_cells": len(planned), "missing_cells": missing, "cells": rows, "ratios": pairs},
indent=2) + "\n")
.github/scripts/oswal-s3-bench.py
+7 -13
View File
@@ -18,11 +18,7 @@
# dependencies = ["boto3>=1.35,<2"]
# ///
"""Plan, image tag, run manifest and bucket helpers of the object store WAL S3 benchmark.
`plan`, `tag` and `manifest` need only the standard library; `probe` and
`cleanup` need boto3 and run under `uv run`.
"""
"""Helpers of the object store WAL S3 benchmark; only `probe` and `cleanup` need boto3 (`uv run`)."""
from __future__ import annotations
@@ -175,22 +171,20 @@ def main() -> int:
for name in ("concurrency", "rows", "batch-rows", "rounds"):
plan.add_argument(f"--{name}", required=True)
plan.add_argument("--smoke", choices=("true", "false"), required=True)
plan.add_argument("--timeout-minutes")
plan.add_argument("--ttl-hours")
plan.add_argument("--output")
for name in ("timeout-minutes", "ttl-hours", "output"):
plan.add_argument(f"--{name}")
commands.add_parser("tag").add_argument("requested")
regions = commands.add_parser("regions")
regions.add_argument("--bucket", required=True)
regions.add_argument("--runner", required=True)
for name in ("bucket", "runner"):
regions.add_argument(f"--{name}", required=True)
manifest = commands.add_parser("manifest")
for name in ("results", "runtime-image", "instance-type", "region", "prefix"):
manifest.add_argument(f"--{name}", required=True)
for name in ("probe", "cleanup"):
bucket = commands.add_parser(name)
bucket.add_argument("--bucket", required=True)
bucket.add_argument("--region", required=True)
bucket.add_argument("--endpoint", default="")
bucket.add_argument("--prefix", required=True)
for option in ("bucket", "region", "prefix"):
bucket.add_argument(f"--{option}", required=True)
args = parser.parse_args()
if args.command == "plan":
if args.timeout_minutes or args.ttl_hours: