rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-13 23:42:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: scrape WAL metrics on failed loads and keep logic out of the benchmark YAML

Browse Source 
Scrape the object store WAL metrics before stopping a cell's server whether
the load succeeded or failed, so failed cells keep their stall, poison and
create failure counters. Build the summary from the plan so that cells which
never ran are listed and leave their ratios undefined. Move budget, region,
runtime image, manifest and credential scan logic from the workflow into the
helper scripts, and drive the cell lifecycle in the tests through fake docker
and curl binaries.

Signed-off-by: jeremyhi <fengjiachun@gmail.com>
This commit is contained in:
jeremyhi
2026-10-12 17:12:12 +08:00
parent 774a4b9432
commit f34b62916f
5 changed files with 314 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/oswal-s3-bench-run.sh
+94 -46
View File
@@ -1,14 +1,40 @@
#!/usr/bin/env bash
# Host side of the object store WAL S3 benchmark (oswal-s3-bench.yml).
# wal-disk DIR mount the instance store at DIR when the host has one
# generate generate the dataset once with the o11ybench runtime
# cells run every cell of $PLAN, each on a fresh GreptimeDB container
# check-runtime IMAGE resolve the o11ybench runtime and check it carries the loader and generator
# pull pull the runtime and GreptimeDB images and record their metadata
# wal-disk DIR mount the instance store at DIR when the host has one
# generate generate the dataset once with the o11ybench runtime
# cells run every cell of $PLAN, each on a fresh GreptimeDB container
# scrub refuse results that contain a credential value
set -euo pipefail
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}"
here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# Same isolation as the o11ybench runtime's own docker run.
runtime=(docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER"
--security-opt no-new-privileges --cap-drop ALL)
runtime() {
docker run --rm --user "$(id -u):$(id -g)" --label "o11ybench.owner=$OWNER" \
--security-opt no-new-privileges --cap-drop ALL "$@"
}
check_runtime() {
local image=$1 resolved
[[ "$image" == public.ecr.aws/* ]] || { echo 'The runtime image must come from AWS ECR Public' >&2; return 1; }
bash "$here/pull-benchmark-image.sh" "$image"
resolved=$(docker image inspect "$image" | jq -er '.[0].RepoDigests[0]')
docker run --rm --entrypoint sh "$resolved" -c \
'test -f scripts/agent-observability/greptimedb.sh && test -f scripts/generate_agent_observations.py' \
|| { echo 'The runtime image lacks the agent observability loader or generator' >&2; return 1; }
printf 'runtime_image=%s\n' "$resolved" >> "${GITHUB_OUTPUT:-/dev/stdout}"
}
pull_images() {
: "${RESULTS:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_TAG:?}"
mkdir -p "$RESULTS/images"
bash "$here/pull-benchmark-image.sh" "$RUNTIME_IMAGE"
bash "$here/pull-benchmark-image.sh" "greptime/greptimedb:$GREPTIMEDB_TAG"
docker image inspect "$RUNTIME_IMAGE" > "$RESULTS/images/runtime.json"
docker image inspect "greptime/greptimedb:$GREPTIMEDB_TAG" > "$RESULTS/images/greptimedb.json"
}
wal_disk() {
local root=$1 device
@@ -24,11 +50,11 @@ wal_disk() {
}
generate() {
local args
mapfile -t args < <(jq -r '.generator_args[]' "$PLAN")
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}"
local args=() arg
while IFS= read -r arg; do args+=("$arg"); done < <(jq -r '.generator_args[]' "$PLAN")
mkdir -p "$CORPUS_DIR"
"${runtime[@]}" --name "$OWNER-generator" \
--mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus" \
runtime --name "$OWNER-generator" --mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus" \
"$RUNTIME_IMAGE" scripts/generate_agent_observations.py "${args[@]}" \
--output /work/corpus/agent_observations.jsonl \
--summary-output /work/corpus/summary.json --evidence-output /work/corpus/evidence.json \
@@ -36,22 +62,50 @@ generate() {
jq -e --argjson rows "$(jq .rows "$PLAN")" '.line_count == $rows' "$CORPUS_DIR/summary.json" >/dev/null
}
# One cell in its own process, so errexit holds inside it.
# Scrape once, after the load succeeded or failed, while the server still runs.
scrape_metrics() {
if curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all"; then
grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true
fi
rm -f "$dir/metrics.all"
}
finish() {
local rc=$?
trap - EXIT
set +e
if [[ -n "$cid" ]]; then
scrape_metrics
docker stop -t 120 "$cid" >/dev/null
docker logs "$cid" > "$dir/server.log" 2>&1
docker rm -fv "$cid" >/dev/null
fi
[[ -z "$wal_dir" ]] || sudo -n rm -rf "$wal_dir"
jq -n --argjson spec "$spec" --arg device "$wal_device" --arg version "$version" \
--arg load_seconds "$load_seconds" --argjson exit_code "$rc" \
'$spec + {wal_device: $device, version: $version, exit_code: $exit_code,
load_wall_seconds: ($load_seconds | tonumber? // null)}' > "$dir/cell.json"
exit "$rc"
}
# One cell in its own process, so errexit holds inside it. The cell's state is
# global because the EXIT trap runs after this function returns.
cell() {
: "${GREPTIMEDB_IMAGE:?}" "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}"
: "${PLAN:?}" "${RESULTS:?}" "${CORPUS_DIR:?}" "${OWNER:?}" "${RUNTIME_IMAGE:?}" "${GREPTIMEDB_IMAGE:?}"
: "${RUN_PREFIX:?}" "${WAL_ROOT:?}" "${S3_BUCKET:?}" "${S3_REGION:?}"
: "${AWS_ACCESS_KEY_ID:?}" "${AWS_SECRET_ACCESS_KEY:?}"
# Globals, not locals: the EXIT trap runs after this function returns.
spec=$1 cid= wal_dir= version= load_seconds=
local concurrency config name
id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec"); config=$(jq -r .config <<< "$spec")
local id concurrency name setting started ended
id=$(jq -r .id <<< "$spec"); concurrency=$(jq -r .concurrency <<< "$spec")
dir="$RESULTS/cells/$id"; name="$OWNER-$id"
mkdir -p "$dir"
# The prefix holds data/ and the WAL's default prefix wal/ side by side.
# The root holds data/ and the WAL's default prefix wal/ side by side.
local settings=(STORAGE__TYPE=S3 "STORAGE__BUCKET=$S3_BUCKET" "STORAGE__REGION=$S3_REGION"
"STORAGE__ROOT=$RUN_PREFIX/$id")
[[ -z "${S3_CONTAINER_ENDPOINT:-}" ]] || settings+=("STORAGE__ENDPOINT=$S3_CONTAINER_ENDPOINT")
mapfile -t -O "${#settings[@]}" settings < <(jq -r '.wal | to_entries[] | "WAL__\(.key)=\(.value)"' <<< "$spec")
local docker_env=() mounts=() setting
while IFS= read -r setting; do settings+=("$setting"); done \
< <(jq -r '.wal | to_entries[] | "WAL__\(.key)=\(.value)"' <<< "$spec")
local docker_env=() mounts=()
for setting in "${settings[@]}"; do
docker_env+=(-e "GREPTIMEDB_STANDALONE__$setting")
printf 'GREPTIMEDB_STANDALONE__%s\n' "$setting" >> "$dir/config.env"
@@ -68,26 +122,10 @@ cell() {
else
wal_device="s3://$S3_BUCKET/$RUN_PREFIX/$id/wal"
fi
finish() {
local rc=$?
trap - EXIT
set +e
if [[ -n "$cid" ]]; then
docker stop -t 120 "$cid" >/dev/null
docker logs "$cid" > "$dir/server.log" 2>&1
docker rm -fv "$cid" >/dev/null
fi
[[ -z "$wal_dir" ]] || sudo -n rm -rf "$wal_dir"
jq -n --argjson spec "$spec" --arg device "$wal_device" --arg version "$version" \
--arg load_seconds "$load_seconds" --argjson exit_code "$rc" \
'$spec + {wal_device: $device, version: $version, exit_code: $exit_code,
load_wall_seconds: ($load_seconds | tonumber? // null)}' > "$dir/cell.json"
exit "$rc"
}
trap finish EXIT
cid=$(docker run -d --init --name "$name" --label "o11ybench.owner=$OWNER" \
--add-host host.docker.internal:host-gateway \
-p 127.0.0.1:4000:4000 -p 127.0.0.1:4001:4001 "${docker_env[@]}" "${mounts[@]}" \
-p 127.0.0.1:4000:4000 -p 127.0.0.1:4001:4001 "${docker_env[@]}" ${mounts[@]+"${mounts[@]}"} \
"$GREPTIMEDB_IMAGE" standalone start --http-addr=0.0.0.0:4000 --grpc-bind-addr=0.0.0.0:4001)
local deadline=$((SECONDS + 300))
until curl --noproxy '*' -fsS --max-time 5 http://127.0.0.1:4000/health >/dev/null 2>&1; do
@@ -97,10 +135,9 @@ cell() {
done
version=$(docker exec "$cid" greptime --version | sed -n 's/^version: //p')
[[ -n "$version" ]]
local started ended
started=$(date +%s.%N)
# Only the load phase, run exactly as run-agent-observability.sh runs it.
"${runtime[@]}" --name "$name-load" --network host \
runtime --name "$name-load" --network host \
--mount "type=bind,src=$CORPUS_DIR,dst=/work/corpus,readonly" --mount "type=bind,src=$dir,dst=/results" \
-e GRPC_ENDPOINT=127.0.0.1:4001 -e "DATA_PROFILE=$(jq -r .data_profile "$PLAN")" -e RESULT_NAME=greptimedb-1 \
-e "ENDPOINT=http://127.0.0.1:4000/v1/sql?db=public&format=greptimedb_v1" \
@@ -112,23 +149,34 @@ cell() {
load_seconds=$(awk -v a="$started" -v b="$ended" 'BEGIN {printf "%.3f", b - a}')
jq -e --argjson rows "$(jq .rows "$PLAN")" '.count_before == 0 and .count_after == $rows' \
"$dir/greptimedb-1/load-result.json" >/dev/null
curl --noproxy '*' -fsS --max-time 30 http://127.0.0.1:4000/metrics > "$dir/metrics.all"
grep -E '^(# (HELP|TYPE) )?greptime_logstore_object_store_wal_' "$dir/metrics.all" > "$dir/metrics.prom" || true
rm "$dir/metrics.all"
}
scrub() {
: "${RESULTS:?}"
[[ -n "${AWS_ACCESS_KEY_ID:-}" && -n "${AWS_SECRET_ACCESS_KEY:-}" && -d "$RESULTS" ]] || return 0
if grep -rqF -e "$AWS_ACCESS_KEY_ID" -e "$AWS_SECRET_ACCESS_KEY" "$RESULTS"; then
echo '::error::A credential value appears in the results; removing them instead of uploading' >&2
rm -rf "$RESULTS"
return 1
fi
}
case "${1:-}" in
check-runtime) check_runtime "${2:?}" ;;
pull) pull_images ;;
wal-disk) wal_disk "${2:?}" ;;
generate) generate ;;
cell) cell "${2:?}" ;;
cells)
mapfile -t specs < <(jq -c '.cells[]' "$PLAN")
: "${PLAN:?}"
failed=0
for spec in "${specs[@]}"; do
echo "::group::cell $(jq -r .id <<< "$spec")"
bash "${BASH_SOURCE[0]}" cell "$spec" || { echo "::error::cell $(jq -r .id <<< "$spec") failed"; failed=1; }
while IFS= read -r spec; do
id=$(jq -r .id <<< "$spec")
echo "::group::cell $id"
bash "${BASH_SOURCE[0]}" cell "$spec" < /dev/null || { echo "::error::cell $id failed"; failed=1; }
echo '::endgroup::'
done
done < <(jq -c '.cells[]' "$PLAN")
exit "$failed" ;;
*) echo "usage: $0 wal-disk DIR | generate | cells" >&2; exit 2 ;;
scrub) scrub ;;
*) echo "usage: $0 check-runtime IMAGE | pull | wal-disk DIR | generate | cells | scrub" >&2; exit 2 ;;
esac
.github/scripts/oswal-s3-bench-summary.py
+22 -6
View File
@@ -87,6 +87,14 @@ def load_cell(path: Path) -> dict:
return row
def missing_cell(spec: dict) -> dict:
"""A planned cell that never recorded a result, so no ratio can use its round."""
row = {key: None for key in ("exit_code", "wal_device", "version", "load_wall_seconds", "rows",
"load_seconds", "rows_per_second", "append_ack_p50_seconds",
"append_ack_p99_seconds", "creates_per_minute", *COUNTERS)}
return row | {key: spec[key] for key in ("id", "round", "concurrency", "config")} | {"missing": True}
def ratios(rows: list) -> list:
"""D/R and E/R per round and their geometric mean over the rounds, per concurrency."""
speed = {(r["round"], r["concurrency"], r["config"]): r["rows_per_second"] for r in rows}
@@ -110,18 +118,21 @@ def fmt(value, digits=0, scale=1.0) -> str:
def render(run: dict, rows: list, pairs: list) -> str:
lines = [f"## Object store WAL against S3 ({'smoke, MinIO' if run.get('smoke') else run.get('instance_type')})", ""]
lines += [f"GreptimeDB `{run.get('greptimedb_image')}` ({rows[0]['version'] if rows else '—'}); "
lines += [f"GreptimeDB `{run.get('greptimedb_image')}` ({next((r['version'] for r in rows if r['version']), '—')}); "
f"{fmt(run.get('rows'))} rows in {run.get('batch_rows')}-row batches; bucket region {run.get('region')}.", ""]
lines += ["| Cell | Rows/s | Load s | Wall s | Rows | Ack p50 ms | Ack p99 ms | Creates | Creates/min "
"| Stalled | Poisoned | Create failures | Conflicts | WAL on |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |"]
for r in rows:
cell = r["id"] if r["exit_code"] == 0 else f"{r['id']} (failed, exit {r['exit_code']})"
if r.get("missing"):
cell = f"{r['id']} (not run)"
else:
cell = r["id"] if r["exit_code"] == 0 else f"{r['id']} (failed, exit {r['exit_code']})"
lines.append(" | ".join([
f"| {cell}", fmt(r["rows_per_second"]), fmt(r["load_seconds"], 1), fmt(r["load_wall_seconds"], 1),
fmt(r["rows"]), fmt(r["append_ack_p50_seconds"], 1, 1000), fmt(r["append_ack_p99_seconds"], 1, 1000),
*(fmt(r[c]) for c in COUNTERS[:1]), fmt(r["creates_per_minute"], 1),
*(fmt(r[c]) for c in COUNTERS[1:]), f"`{r['wal_device']}` |"]))
*(fmt(r[c]) for c in COUNTERS[1:]), f"`{r['wal_device'] or '—'}` |"]))
lines += ["", "| Concurrency | Ratio | Per round | Geometric mean |", "| ---: | --- | --- | ---: |"]
for p in pairs:
lines.append(f"| {p['concurrency']} | {p['ratio']} | {', '.join(fmt(v, 3) for v in p['per_round'])} "
@@ -137,11 +148,16 @@ def main() -> None:
run_path = args.root / "run.json"
run = json.loads(run_path.read_text()) if run_path.exists() else {}
plan_path = args.root / "plan.json"
order = [cell["id"] for cell in json.loads(plan_path.read_text())["cells"]] if plan_path.exists() else []
planned = json.loads(plan_path.read_text())["cells"] if plan_path.exists() else []
cells = {path.name: load_cell(path) for path in (args.root / "cells").glob("*") if (path / "cell.json").exists()}
rows = [cells[i] for i in order if i in cells] + [cells[i] for i in sorted(cells) if i not in order]
# The plan, not what happened to run, defines the expected cells and rounds.
rows = [cells.get(spec["id"]) or missing_cell(spec) for spec in planned]
rows += [cells[i] for i in sorted(cells) if i not in {spec["id"] for spec in planned}]
pairs = ratios(rows)
(args.root / "summary.json").write_text(json.dumps({"run": run, "cells": rows, "ratios": pairs}, indent=2) + "\n")
missing = [r["id"] for r in rows if r.get("missing")]
(args.root / "summary.json").write_text(json.dumps(
{"run": run, "planned_cells": len(planned), "missing_cells": missing, "cells": rows, "ratios": pairs},
indent=2) + "\n")
print(render(run, rows, pairs) if rows else "No benchmark cell completed.\n")
.github/scripts/oswal-s3-bench.py
+58 -7
View File
@@ -18,10 +18,10 @@
# dependencies = ["boto3>=1.35,<2"]
# ///
"""Plan, image tag and bucket helpers of the object store WAL S3 benchmark.
"""Plan, image tag, run manifest and bucket helpers of the object store WAL S3 benchmark.
`plan` and `tag` need only the standard library; `probe` and `cleanup` need
boto3 and run under `uv run`.
`plan`, `tag` and `manifest` need only the standard library; `probe` and
`cleanup` need boto3 and run under `uv run`.
"""
from __future__ import annotations
@@ -33,6 +33,7 @@ import re
import sys
import time
import urllib.request
from pathlib import Path
SMOKE_ROWS = 50_000
# The o11ybench data profiles; a matching row count generates the canonical dataset.
@@ -46,10 +47,11 @@ WAL_CONFIGS = {
ORDERS = (("R", "D", "E"), ("E", "D", "R"))
NIGHTLY = re.compile(r"v\d+\.\d+\.\d+(?:-[0-9A-Za-z.]+)?-nightly-(\d{8})")
TAG = re.compile(r"[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}")
DIGEST = re.compile(r".+@sha256:[0-9a-f]{64}")
def append_output(name: str, value: str) -> None:
if path := os.environ.get("GITHUB_OUTPUT"):
def append_github(variable: str, name: str, value: str) -> None:
if path := os.environ.get(variable):
with open(path, "a") as output:
output.write(f"{name}={value}\n")
@@ -60,6 +62,30 @@ def positive(name: str, value: str, upper: int) -> int:
return int(value)
def validate_budgets(timeout_minutes: str, ttl_hours: str) -> None:
timeout, ttl = positive("timeout", timeout_minutes, 360), positive("ttl", ttl_hours, 168)
# Provisioning, teardown and queueing need two hours beyond the benchmark job.
if ttl * 60 < timeout + 120:
raise ValueError(f"janitor TTL {ttl}h must cover the {timeout}-minute timeout plus 2 hours")
def check_regions(bucket: str, runner: str) -> None:
if not bucket or bucket != runner:
raise ValueError(f"bucket region {bucket!r} differs from runner region {runner!r}; "
"a cross-region bucket measures the WAN, not S3")
def build_manifest(plan: dict, greptimedb_inspect: list, runtime_image: str,
instance_type: str, region: str, prefix: str) -> dict:
image = next((d for d in greptimedb_inspect[0].get("RepoDigests") or [] if DIGEST.fullmatch(d)), None)
if image is None:
raise ValueError("the GreptimeDB image has no sha256 repository digest")
return {"smoke": plan["smoke"], "rows": plan["rows"], "rounds": plan["rounds"],
"concurrency": plan["concurrency"], "batch_rows": plan["batch_rows"],
"greptimedb_image": image, "runtime_image": runtime_image,
"instance_type": instance_type, "region": region, "prefix": prefix}
def build_plan(concurrency: str, rows: str, batch_rows: str, rounds: str, smoke: bool) -> dict:
levels = [positive("concurrency", level.strip(), 256) for level in concurrency.split(",")]
if len(set(levels)) != len(levels):
@@ -149,8 +175,16 @@ def main() -> int:
for name in ("concurrency", "rows", "batch-rows", "rounds"):
plan.add_argument(f"--{name}", required=True)
plan.add_argument("--smoke", choices=("true", "false"), required=True)
plan.add_argument("--timeout-minutes")
plan.add_argument("--ttl-hours")
plan.add_argument("--output")
commands.add_parser("tag").add_argument("requested")
regions = commands.add_parser("regions")
regions.add_argument("--bucket", required=True)
regions.add_argument("--runner", required=True)
manifest = commands.add_parser("manifest")
for name in ("results", "runtime-image", "instance-type", "region", "prefix"):
manifest.add_argument(f"--{name}", required=True)
for name in ("probe", "cleanup"):
bucket = commands.add_parser(name)
bucket.add_argument("--bucket", required=True)
@@ -159,6 +193,8 @@ def main() -> int:
bucket.add_argument("--prefix", required=True)
args = parser.parse_args()
if args.command == "plan":
if args.timeout_minutes or args.ttl_hours:
validate_budgets(args.timeout_minutes or "", args.ttl_hours or "")
result = build_plan(args.concurrency, args.rows, args.batch_rows, args.rounds, args.smoke == "true")
if args.output:
with open(args.output, "w") as output:
@@ -166,8 +202,19 @@ def main() -> int:
print(f"{len(result['cells'])} cells, {result['rows']} rows, profile {result['data_profile']}")
elif args.command == "tag":
tag = resolve_tag(args.requested)
append_output("greptimedb_tag", tag)
append_github("GITHUB_OUTPUT", "greptimedb_tag", tag)
print(f"GreptimeDB image tag: {tag}")
elif args.command == "regions":
check_regions(args.bucket, args.runner)
print(f"bucket and runner both in {args.runner}")
elif args.command == "manifest":
results = Path(args.results)
manifest = build_manifest(json.loads((results / "plan.json").read_text()),
json.loads((results / "images" / "greptimedb.json").read_text()),
args.runtime_image, args.instance_type, args.region, args.prefix)
(results / "run.json").write_text(json.dumps(manifest, indent=2) + "\n")
append_github("GITHUB_ENV", "GREPTIMEDB_IMAGE", manifest["greptimedb_image"])
print(f"GreptimeDB image: {manifest['greptimedb_image']}")
else:
if not re.fullmatch(r"oswal-bench/[0-9]+-[0-9]+", args.prefix):
raise ValueError(f"refusing a prefix outside oswal-bench/<run>: {args.prefix!r}")
@@ -176,4 +223,8 @@ def main() -> int:
if __name__ == "__main__":
sys.exit(main())
try:
sys.exit(main())
except ValueError as error:
print(f"::error::{error}", file=sys.stderr)
sys.exit(1)