ci: add OTLP trace ingestion regression testing (#8631)

* chore: update CI config

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add CI

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: update CI config

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* ci: report otelgen runner diagnostics

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* chore: add script to draw result diagram

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

src/cmd/src/bin/query_perf_fixture/case.rs

@@ -13,7 +13,7 @@
 // limitations under the License.
 
 use std::collections::HashMap;
-use std::num::NonZeroUsize;
+use std::num::{NonZeroU64, NonZeroUsize};
 
 use clap::ValueEnum;
 use serde::{Deserialize, Serialize};
@@ -30,6 +30,8 @@ pub(super) enum Scenario {
     DirectReadableSst(DirectReadableSstScenario),
     #[serde(rename = "prom_remote_write_then_query")]
     PromRemoteWriteThenQuery(PromRemoteWriteThenQueryScenario),
+    #[serde(rename = "otlp_trace_load")]
+    OtlpTraceLoad(OtlpTraceLoadScenario),
 }
 
 #[derive(Debug, Deserialize, Serialize)]
@@ -49,6 +51,33 @@ pub(super) struct PromRemoteWriteThenQueryScenario {
     pub(super) remote_write: PromRemoteWritePlan,
 }
 
+#[derive(Debug, Deserialize, Serialize)]
+pub(super) struct OtlpTraceLoadScenario {
+    pub(super) load: OtlpTraceLoadPlan,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub(super) struct OtlpTraceLoadPlan {
+    pub(super) database: String,
+    pub(super) table: String,
+    pub(super) pipeline: String,
+    pub(super) duration_seconds: NonZeroU64,
+    pub(super) warmup_seconds: u64,
+    pub(super) rate: NonZeroU64,
+    pub(super) workers: NonZeroUsize,
+    pub(super) exporter_shards: NonZeroUsize,
+    pub(super) workload: String,
+    pub(super) visibility_timeout_seconds: NonZeroU64,
+    pub(super) thresholds: OtlpTraceLoadThresholds,
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+pub(super) struct OtlpTraceLoadThresholds {
+    pub(super) max_candidate_throughput_regression_pct: f64,
+    pub(super) max_candidate_mean_latency_regression_pct: f64,
+    pub(super) max_failure_count: u64,
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub(super) struct PromRemoteWritePlan {
     #[serde(default = "default_database")]
@@ -412,6 +441,7 @@ impl Scenario {
         match self {
             Scenario::DirectReadableSst(_) => "direct_readable_sst",
             Scenario::PromRemoteWriteThenQuery(_) => "prom_remote_write_then_query",
+            Scenario::OtlpTraceLoad(_) => "otlp_trace_load",
         }
     }
 

src/cmd/src/bin/query_perf_fixture/mod.rs

@@ -125,6 +125,28 @@ fn run_plan(args: PlanArgs) -> Result<(), Box<dyn std::error::Error>> {
             return Err("scenario.remote_write.read_bench requires scenario.remote_write.storage.inspect = true".into());
         }
     }
+    if let Scenario::OtlpTraceLoad(s) = &case.scenario {
+        if s.load.warmup_seconds >= s.load.duration_seconds.get() {
+            return Err("scenario.load.warmup_seconds must be less than duration_seconds".into());
+        }
+        for (name, value) in [
+            (
+                "max_candidate_throughput_regression_pct",
+                s.load.thresholds.max_candidate_throughput_regression_pct,
+            ),
+            (
+                "max_candidate_mean_latency_regression_pct",
+                s.load.thresholds.max_candidate_mean_latency_regression_pct,
+            ),
+        ] {
+            if !value.is_finite() || value < 0.0 {
+                return Err(format!(
+                    "scenario.load.thresholds.{name} must be a finite non-negative number"
+                )
+                .into());
+            }
+        }
+    }
     println!(
         "{}",
         serde_json::to_string_pretty(&json!({"schema_version": 1, "scenario": case.scenario}))?