ci: add OTLP trace ingestion regression testing (#8631)

* chore: update CI config Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: add CI Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: update CI config Signed-off-by: shuiyisong <xixing.sys@gmail.com> * ci: report otelgen runner diagnostics Signed-off-by: shuiyisong <xixing.sys@gmail.com> * chore: add script to draw result diagram Signed-off-by: shuiyisong <xixing.sys@gmail.com> --------- Signed-off-by: shuiyisong <xixing.sys@gmail.com>
2026-09-07 22:18:57 +00:00 · 2026-07-26 15:08:27 +00:00
parent cbabf8a53d
commit f5f5d468bb
15 changed files with 924 additions and 23 deletions
@@ -1,3 +1,17 @@
+FROM golang:1.23@sha256:e87b2a5f6df2dff71ea330d55d54f4979eb380ae58a7e3aabc9d53121243e689 AS otelgen
+
+ARG OTELGEN_COMMIT=863a3f395d062c7322cc1de08a38774b7fdaa6c8
+
+RUN git init /tmp/otelgen \
+    && git -C /tmp/otelgen remote add origin https://github.com/WenyXu/otelgen.git \
+    && git -C /tmp/otelgen fetch --depth=1 origin "${OTELGEN_COMMIT}" \
+    && test "$(git -C /tmp/otelgen rev-parse FETCH_HEAD)" = "${OTELGEN_COMMIT}" \
+    && git -C /tmp/otelgen checkout --detach FETCH_HEAD \
+    && cd /tmp/otelgen \
+    && CGO_ENABLED=0 go build -trimpath \
+        -ldflags "-s -w -X main.version=${OTELGEN_COMMIT} -X main.commit=${OTELGEN_COMMIT}" \
+        -o /usr/local/bin/otelgen ./cmd/otelgen
+
 FROM ghcr.io/astral-sh/uv@sha256:3d868e555f8f1dbc324afa005066cd11e1053fc4743b9808ca8025283e65efa5 AS uv
 FROM ghcr.io/actions/actions-runner@sha256:08c30b0a7105f64bddfc485d2487a22aa03932a791402393352fdf674bda2c29

@@ -43,6 +57,7 @@ RUN apt-get update \
    && rm -rf /var/lib/apt/lists/*

 COPY --from=uv /uv /uvx /usr/local/bin/
+COPY --from=otelgen /usr/local/bin/otelgen /usr/local/bin/otelgen

 RUN curl --fail --location --silent --show-error \
        --output /tmp/sccache.tar.gz \
@@ -68,10 +68,15 @@ operator before any canary.
 ## Runner image and workflow tools

 Build and push the derived runner image; it preserves the official
-`/home/runner/run.sh` entrypoint and supplies CI tools needed at runtime:
+`/home/runner/run.sh` entrypoint and supplies CI tools needed at runtime. The
+image builds `otelgen` from
+[`WenyXu/otelgen`](https://github.com/WenyXu/otelgen) commit
+[`863a3f395d062c7322cc1de08a38774b7fdaa6c8`](https://github.com/WenyXu/otelgen/commit/863a3f395d062c7322cc1de08a38774b7fdaa6c8)
+so trace cases do not download or compile tools during a benchmark run:

 ```bash
 docker build \
+  --platform linux/amd64 \
  -f .github/runner-scale-sets/query-regression/Dockerfile \
  -t greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner:latest \
  .github/runner-scale-sets/query-regression
@@ -79,9 +84,11 @@ docker build \
 docker push greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner:latest
 ```

-Deploy by digest, not mutable tag, by updating `values-8-cores.yaml` after a
-rebuild. If the registry is private, use a dedicated read-only pull secret only
-as `imagePullSecrets`; never expose registry credentials to runner containers.
+Deploy by digest, not mutable tag, by updating both image references in
+`values-8-cores.yaml` after a rebuild. Update `RUNNER_IMAGE_DIGEST` and bump
+`RUNNER_IMAGE_EPOCH` in `query-regression.yml` at the same time. If the registry
+is private, use a dedicated read-only pull secret only as `imagePullSecrets`;
+never expose registry credentials to runner containers.
 Both digest-pinned init and runner containers use `IfNotPresent`: the immutable
 digest makes a cached image safe and avoids adding a registry dependency to every
 runner startup.
@@ -94,7 +101,8 @@ credentials or secrets must never be placed in a ConfigMap.

 Before builds, the workflow asserts UID/GID 1001 and exact image tool versions:
 `libprotoc 3.21.12`, `uv 0.11.26`, `mold 2.30.0`, `Python 3.12.3`, `sccache
-0.16.0`, root-owned `rustup 1.29.0`, and the image-baked
+0.16.0`, `otelgen` commit `863a3f395d062c7322cc1de08a38774b7fdaa6c8`,
+root-owned `rustup 1.29.0`, and the image-baked
 `nightly-2026-03-21` Rust toolchain. Rustup, Cargo, and Rustc must resolve from
 `/opt/cargo/bin`; the runner cannot write `/opt/rustup` or `/opt/cargo/bin`.
 Protobuf well-known includes, including `google/protobuf/any.proto` and
@@ -27,7 +27,7 @@ template:
          claimName: query-regression-build-cache
    initContainers:
      - name: initialize-build-cache
-        image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:2436278bf0756700c718927907cdbedac64d55c593f80b515acb44dec8b02cc1
+        image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:e713b294e23b7e15184e558866c90025e59930033e72c97650dbc7f1ca022d11
        imagePullPolicy: IfNotPresent
        command:
          - /bin/sh
@@ -65,7 +65,7 @@ template:
            mountPath: /cache
    containers:
      - name: runner
-        image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:2436278bf0756700c718927907cdbedac64d55c593f80b515acb44dec8b02cc1
+        image: greptime-registry.cn-hangzhou.cr.aliyuncs.com/greptime/greptimedb-query-regression-runner@sha256:e713b294e23b7e15184e558866c90025e59930033e72c97650dbc7f1ca022d11
        imagePullPolicy: IfNotPresent
        command: ["/home/runner/run.sh"]
        env:
@@ -118,6 +118,8 @@ def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int:
    ]
    if parse_bool(args.allow_large_fixture):
        cmd.append("--allow-large-fixture")
+    if args.otelgen_bin is not None:
+        cmd.extend(["--otelgen-bin", str(args.otelgen_bin)])

    print(f"::group::Query regression case: {case_path}", flush=True)
    try:
@@ -159,6 +161,7 @@ def main() -> int:
        type=Path,
        default=configured_path(os.environ.get("FIXTURE_GENERATOR")),
    )
+    parser.add_argument("--otelgen-bin", type=Path, default=configured_path(os.environ.get("OTELGEN_BIN")))
    parser.add_argument("--cargo-profile", default=os.environ.get("CARGO_PROFILE", "nightly"))
    parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path)
    parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300"))
@@ -32,15 +32,25 @@ def fmt_ms(value: Any) -> str:
        return "N/A"


+def fmt_pct(value: Any) -> str:
+    formatted = fmt_ms(value)
+    return formatted if formatted == "N/A" else f"{formatted}%"
+
+
 def esc(value: Any) -> str:
    text = "N/A" if value is None else str(value)
    return text.replace("|", "\\|").replace("\n", " ")


 def status_emoji(status: str | None) -> str:
-    return {"ok": "✅", "measured": "✅", "failed": "❌", "planned": "📝", "fixture-ready": "🧪"}.get(
-        status or "", "⚠️"
-    )
+    return {
+        "ok": "✅",
+        "passed": "✅",
+        "measured": "✅",
+        "failed": "❌",
+        "planned": "📝",
+        "fixture-ready": "🧪",
+    }.get(status or "", "⚠️")


 def measurement_map(target: dict[str, Any]) -> dict[str, dict[str, Any]]:
@@ -114,6 +124,50 @@ def comparison_table(targets: list[dict[str, Any]], thresholds: list[dict[str, A
    return "\n".join(rows)


+def otlp_comparison_table(targets: list[dict[str, Any]], thresholds: list[dict[str, Any]]) -> str:
+    rows = [
+        "| Target | Accepted spans | Table rows | Spans/s | Mean HTTP latency ms | Failures |",
+        "| --- | ---: | ---: | ---: | ---: | ---: |",
+    ]
+    for target in targets:
+        metrics = target.get("metrics") or {}
+        visibility = target.get("visibility") or {}
+        rows.append(
+            "| {target} | {accepted} | {stored} | {rate} | {latency} | {failures} |".format(
+                target=esc(target.get("name")),
+                accepted=esc(metrics.get("accepted_spans")),
+                stored=esc(visibility.get("observed_rows")),
+                rate=fmt_ms(metrics.get("accepted_spans_per_second")),
+                latency=fmt_ms(metrics.get("mean_http_latency_ms")),
+                failures=esc(metrics.get("failure_count")),
+            )
+        )
+
+    rows.extend(
+        [
+            "",
+            "| Threshold | Scope | Actual | Limit | Status | Reason |",
+            "| --- | --- | ---: | ---: | --- | --- |",
+        ]
+    )
+    for threshold in thresholds:
+        is_pct = "actual_pct" in threshold or "limit_pct" in threshold
+        actual = fmt_pct(threshold.get("actual_pct")) if is_pct else esc(threshold.get("actual"))
+        limit = fmt_pct(threshold.get("limit_pct")) if is_pct else esc(threshold.get("limit"))
+        rows.append(
+            "| {threshold} | {scope} | {actual} | {limit} | {status} {raw} | {reason} |".format(
+                threshold=esc(threshold.get("threshold")),
+                scope=esc(threshold.get("target") or "base vs candidate"),
+                actual=actual,
+                limit=limit,
+                status=status_emoji(threshold.get("status")),
+                raw=esc(threshold.get("status")),
+                reason=esc(threshold.get("reason")),
+            )
+        )
+    return "\n".join(rows)
+
+
 def build_markdown(
    report: dict[str, Any],
    report_path: Path,
@@ -141,7 +195,12 @@ def build_markdown(

    targets = report.get("targets") or []
    lines.extend(["", "### Targets", "", target_table(targets)])
-    lines.extend(["", "### Query comparison", "", comparison_table(targets, report.get("thresholds") or [])])
+    if (report.get("scenario") or {}).get("kind") == "otlp_trace_load":
+        lines.extend(
+            ["", "### OTLP trace comparison", "", otlp_comparison_table(targets, report.get("thresholds") or [])]
+        )
+    else:
+        lines.extend(["", "### Query comparison", "", comparison_table(targets, report.get("thresholds") or [])])

    not_enforced = [t for t in report.get("thresholds") or [] if t.get("status") == "not_enforced"]
    if not_enforced:
@@ -286,6 +286,15 @@ jobs:
          mold_version="$(mold --version)"
          [[ "${mold_version}" =~ ^mold[[:space:]]2\.30\.0([[:space:]]|$) ]]
          [[ "$(python3 --version)" == "Python 3.12.3" ]]
+          otelgen_path="$(command -v otelgen || true)"
+          otelgen_version="<unavailable>"
+          if [[ -n "${otelgen_path}" ]]; then
+            otelgen_version="$("${otelgen_path}" --version 2>&1 || true)"
+          fi
+          printf 'otelgen path: %s\n' "${otelgen_path:-<missing>}"
+          printf 'otelgen version: %s\n' "${otelgen_version}"
+          [[ "${otelgen_path}" == "/usr/local/bin/otelgen" ]]
+          [[ "${otelgen_version}" == *"863a3f395d062c7322cc1de08a38774b7fdaa6c8"* ]]
          sccache_version="$(sccache --version)"
          [[ "${sccache_version}" =~ ^sccache[[:space:]]0\.16\.0([[:space:]]|$) ]]
          [[ "$(command -v rustup)" == "/opt/cargo/bin/rustup" ]]
@@ -324,8 +333,8 @@ jobs:
          readonly EXPECTED_CACHE_META="/home/runner/query-regression-cache-meta"
          readonly EXPECTED_SCCACHE_DIR="/home/runner/.cache/sccache"
          readonly EXPECTED_RUSTC_WRAPPER="/usr/local/bin/sccache"
-          readonly RUNNER_IMAGE_DIGEST="sha256:2436278bf0756700c718927907cdbedac64d55c593f80b515acb44dec8b02cc1"
-          readonly RUNNER_IMAGE_EPOCH="4"
+          readonly RUNNER_IMAGE_DIGEST="sha256:e713b294e23b7e15184e558866c90025e59930033e72c97650dbc7f1ca022d11"
+          readonly RUNNER_IMAGE_EPOCH="5"

          require_expected_root() {
            local name="$1"
@@ -558,9 +567,12 @@ jobs:
          git reset --hard "${VERIFIED_CANDIDATE_SHA}"
          git clean -ffdx

-      - name: Test query regression runner lifecycle
+      - name: Test query regression tooling
        working-directory: src
-        run: uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py
+        run: |
+          uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py
+          uv run --no-project python tests/perf/test_query_regression_runner_otlp_trace_load.py
+          uv run --no-project python tests/perf/test_query_regression_summary_otlp.py

      - name: Build candidate greptime and fixture generators
        working-directory: src
@@ -594,11 +606,13 @@ jobs:
          BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime
          CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime
          FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture
+          OTELGEN_BIN: /usr/local/bin/otelgen
+          SUMMARY_SCRIPT: ${{ github.event_name == 'pull_request' && 'query-regression-trusted-scripts/query-regression-summary.py' || 'src/.github/scripts/query-regression-summary.py' }}
        run: >-
          uv run --no-project python src/.github/scripts/query-regression-run.py
          --base-src src
          --candidate-src src
-          --summary-script query-regression-trusted-scripts/query-regression-summary.py
+          --summary-script "${SUMMARY_SCRIPT}"

      - name: Write PR metadata for trusted comment workflow
        if: ${{ always() && github.event_name == 'pull_request' }}
@@ -622,6 +636,7 @@ jobs:
            query-regression-work/**/base/report.json
            query-regression-work/**/candidate/report.json
            query-regression-work/**/logs/**
+            query-regression-work/**/otelgen/**
            query-regression-summary.md
          if-no-files-found: warn
          retention-days: 7