feat: track prometheus HTTP API's query latency (#4458)

* feat: track prometheus HTTP API's query latency Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * update grafana config Signed-off-by: Ruihang Xia <waynestxia@gmail.com> * chore: Update src/servers/src/metrics.rs --------- Signed-off-by: Ruihang Xia <waynestxia@gmail.com> Co-authored-by: Yingwen <realevenyag@gmail.com>
2026-01-07 22:02:56 +00:00 · 2024-07-29 23:00:09 +08:00
parent f0c953f84a
commit 1895a5478b
3 changed files with 158 additions and 5 deletions
--- a/grafana/greptimedb.json
+++ b/grafana/greptimedb.json
@@ -679,7 +679,7 @@
          "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))",
          "hide": false,
          "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "sql-{{db}}-p95",
          "range": true,
          "refId": "SQL P95"
        },
@@ -692,7 +692,7 @@
          "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_sql_elapsed_bucket[$__rate_interval])))",
          "hide": false,
          "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "sql-{{db}}-p99",
          "range": true,
          "refId": "SQL P99"
        },
@@ -705,7 +705,7 @@
          "expr": "histogram_quantile(0.95, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))",
          "hide": false,
          "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "promstore-read-{{db}}-p95",
          "range": true,
          "refId": "PromStore Read P95"
        },
@@ -718,9 +718,35 @@
          "expr": "histogram_quantile(0.99, sum by(le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket[$__rate_interval])))",
          "hide": false,
          "instant": false,
-          "legendFormat": "__auto",
+          "legendFormat": "promstore-read-{{db}}-p99",
          "range": true,
          "refId": "PromStore Read P99"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-1}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.95, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "prom-promql-{{db}}-{{method}}-p95",
+          "range": true,
+          "refId": "Prometheus PromQL P95"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-1}"
+          },
+          "editorMode": "code",
+          "expr": "histogram_quantile(0.99, sum by(le, db, method) (rate(greptime_servers_http_prometheus_promql_elapsed_bucket[$__rate_interval])))",
+          "hide": false,
+          "instant": false,
+          "legendFormat": "prom-promql-{{db}}-{{method}}-p99",
+          "range": true,
+          "refId": "Prometheus PromQL P99"
        }
      ],
      "title": "HTTP query elapsed",
@@ -1090,6 +1116,106 @@
      "title": "gRPC insert elapsed",
      "type": "timeseries"
    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS-1}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 0,
+            "gradientMode": "none",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 12,
+        "y": 18
+      },
+      "id": 36,
+      "interval": "1s",
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS-1}"
+          },
+          "disableTextWrap": false,
+          "editorMode": "code",
+          "expr": "sum by(db) (rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+          "fullMetaSearch": false,
+          "includeNullMetadata": false,
+          "instant": false,
+          "legendFormat": "{{db}}",
+          "range": true,
+          "refId": "A",
+          "useBackend": false
+        }
+      ],
+      "title": "Logs ingest rate (number of lines)",
+      "type": "timeseries"
+    },
    {
      "collapsed": false,
      "gridPos": {
@@ -2626,4 +2752,4 @@
  "uid": "e7097237-669b-4f8d-b751-13067afbfb68",
  "version": 14,
  "weekStart": ""
-}
+}
--- a/src/servers/src/http/prometheus.rs
+++ b/src/servers/src/http/prometheus.rs
@@ -193,6 +193,10 @@ pub async fn instant_query(
        query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
    }

+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "instant_query"])
+        .start_timer();
+
    let result = handler.do_query(&prom_query, query_ctx).await;
    let (metric_name, result_type) = match retrieve_metric_name_and_result_type(&prom_query.query) {
        Ok((metric_name, result_type)) => (metric_name.unwrap_or_default(), result_type),
@@ -242,6 +246,10 @@ pub async fn range_query(
        query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
    }

+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "range_query"])
+        .start_timer();
+
    let result = handler.do_query(&prom_query, query_ctx).await;
    let metric_name = match retrieve_metric_name_and_result_type(&prom_query.query) {
        Err(err) => {
@@ -316,6 +324,10 @@ pub async fn labels_query(
        queries = form_params.matches.0;
    }

+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "labels_query"])
+        .start_timer();
+
    // Fetch all tag columns. It will be used as white-list for tag names.
    let mut labels = match get_all_column_names(&catalog, &schema, &handler.catalog_manager()).await
    {
@@ -687,6 +699,10 @@ pub async fn label_values_query(
    let (catalog, schema) = get_catalog_schema(&params.db, &query_ctx);
    let query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);

+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "label_values_query"])
+        .start_timer();
+
    if label_name == METRIC_NAME_LABEL {
        let mut table_names = match handler
            .catalog_manager()
@@ -968,6 +984,10 @@ pub async fn series_query(
        query_ctx = try_update_catalog_schema(query_ctx, &catalog, &schema);
    }

+    let _timer = crate::metrics::METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED
+        .with_label_values(&[query_ctx.get_db_string().as_str(), "series_query"])
+        .start_timer();
+
    let mut series = Vec::new();
    let mut merge_map = HashMap::new();
    for query in queries {
--- a/src/servers/src/metrics.rs
+++ b/src/servers/src/metrics.rs
@@ -120,6 +120,13 @@ lazy_static! {
        &[METRIC_DB_LABEL]
    )
    .unwrap();
+    /// Http prometheus endpoint query duration per database.
+    pub static ref METRIC_HTTP_PROMETHEUS_PROMQL_ELAPSED: HistogramVec = register_histogram_vec!(
+        "greptime_servers_http_prometheus_promql_elapsed",
+        "servers http prometheus promql elapsed",
+        &[METRIC_DB_LABEL, METRIC_METHOD_LABEL]
+    )
+    .unwrap();
    pub static ref METRIC_HTTP_OPENTELEMETRY_METRICS_ELAPSED: HistogramVec =
        register_histogram_vec!(
            "greptime_servers_http_otlp_metrics_elapsed",