From 9ed84cc26f3d5c441a4054b093c35e289c86b52e Mon Sep 17 00:00:00 2001 From: Yingwen Date: Tue, 16 Jun 2026 15:13:04 +0800 Subject: [PATCH] fix: improve Grafana metrics dashboards (#8298) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: initial changes Signed-off-by: evenyag * feat: improve troubleshooting dashboard Signed-off-by: evenyag * chore: rm troubleshooting-dashboard.md Signed-off-by: evenyag * chore: optimize metrics dashboard Signed-off-by: evenyag * docs: move troubleshooting-dashboard.md Signed-off-by: evenyag * chore: move mito gc duration panel Signed-off-by: evenyag * chore: cleanup the dashboard - Overview trend panels are now aggregate-only: - Total Ingestion Rate Trend - Total Query Rate Trend - Protocol breakdowns remain in Ingestion and Queries. - Mito Backpressure and Failures no longer duplicates scan/GC signals. - Removed Write Stall per Instance. - Split Object Store and WAL into collapsed Object Store and collapsed WAL. - Moved WAL/logstore panels out of Storage into WAL. - Normalized OpenDAL “other request” matchers. - Normalized trigger elapsed p99/p75/avg aggregation. - Regenerated standalone JSON and dashboard YAML/Markdown. - Updated docs/troubleshooting-dashboard.md. Signed-off-by: evenyag * fix: rearrange metasrv dashboard panels Signed-off-by: evenyag * feat: improve troubleshooting dashboard layout Signed-off-by: evenyag * docs: remove obsolete troubleshooting dashboard doc Signed-off-by: evenyag * fix: correct cluster dashboard panel queries (missing _bucket, raw counters, rate normalization) Signed-off-by: evenyag * fix: correct trigger panel datasource, collapse flush/compaction, split request latency panels Signed-off-by: evenyag * fix: update grafana metrics dashboard panels Signed-off-by: evenyag * fix: correct Grafana dashboard units Signed-off-by: evenyag * chore: regenerate Grafana dashboards Signed-off-by: evenyag * fix: use throughput unit for index IO bytes Signed-off-by: evenyag --------- Signed-off-by: evenyag --- .../dashboards/metrics/cluster/dashboard.json | 17900 ++++++++++----- .../dashboards/metrics/cluster/dashboard.md | 232 +- .../dashboards/metrics/cluster/dashboard.yaml | 2010 +- .../metrics/standalone/dashboard.json | 17932 +++++++++++----- .../metrics/standalone/dashboard.md | 260 +- .../metrics/standalone/dashboard.yaml | 2116 +- grafana/scripts/gen-dashboards.sh | 6 +- 7 files changed, 27590 insertions(+), 12866 deletions(-) diff --git a/grafana/dashboards/metrics/cluster/dashboard.json b/grafana/dashboards/metrics/cluster/dashboard.json index 01253bd83b6..507d9312e6b 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.json +++ b/grafana/dashboards/metrics/cluster/dashboard.json @@ -36,7 +36,7 @@ "x": 0, "y": 0 }, - "id": 279, + "id": 393, "panels": [], "title": "Overview", "type": "row" @@ -68,12 +68,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, "x": 0, - "y": 1 + "y": 1, + "w": 3, + "h": 4 }, - "id": 265, + "id": 380, "options": { "colorMode": "value", "graphMode": "none", @@ -142,12 +142,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 2, "x": 3, - "y": 1 + "y": 1, + "w": 3, + "h": 4 }, - "id": 239, + "id": 381, "options": { "colorMode": "value", "graphMode": "area", @@ -227,12 +227,12 @@ "overrides": [] }, "gridPos": { - "h": 4, + "x": 6, + "y": 1, "w": 4, - "x": 5, - "y": 1 + "h": 4 }, - "id": 249, + "id": 382, "options": { "colorMode": "value", "graphMode": "none", @@ -270,16 +270,18 @@ }, { "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, - "description": "Total number of data file size.", + "description": "Total query API call rate across MySQL, PostgreSQL, and PromQL frontends.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "fieldMinMax": false, "mappings": [], + "max": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -288,20 +290,20 @@ } ] }, - "unit": "decbytes" + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 4, + "x": 10, + "y": 1, "w": 4, - "x": 9, - "y": 1 + "h": 4 }, - "id": 248, + "id": 383, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -310,7 +312,8 @@ "lastNotNull" ], "fields": "", - "values": false + "limit": 1, + "values": true }, "showPercentChange": false, "textMode": "auto", @@ -319,72 +322,62 @@ "pluginVersion": "11.6.0", "targets": [ { - "dataset": "information_schema", "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "queries", + "range": true, + "refId": "A" } ], - "title": "Total Storage Size", + "title": "Total Query Rate", "type": "stat" }, { "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, - "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", + "description": "Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "fieldMinMax": false, "mappings": [], + "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" + }, + { + "color": "red", + "value": 0 } ] }, - "unit": "sishort" + "unit": "eps" }, "overrides": [] }, "gridPos": { - "h": 4, + "x": 14, + "y": 1, "w": 4, - "x": 13, - "y": 1 + "h": 4 }, - "id": 254, + "id": 384, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -393,7 +386,8 @@ "lastNotNull" ], "fields": "", - "values": false + "limit": 1, + "values": true }, "showPercentChange": false, "textMode": "auto", @@ -402,36 +396,94 @@ "pluginVersion": "11.6.0", "targets": [ { - "dataset": "information_schema", "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "expr": "sum(rate(greptime_servers_error{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "errors", + "range": true, + "refId": "A" } ], - "title": "Total Rows", + "title": "User-facing Error Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Process restarts over the selected time range across GreptimeDB roles.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "id": 385, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 1, + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(changes(process_start_time_seconds[$__range]))", + "hide": false, + "instant": false, + "legendFormat": "restarts", + "range": true, + "refId": "A" + } + ], + "title": "Recent Restarts", "type": "stat" }, { @@ -462,12 +514,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 5, "x": 0, - "y": 5 + "y": 5, + "w": 6, + "h": 4 }, - "id": 243, + "id": 386, "options": { "colorMode": "value", "graphMode": "area", @@ -634,12 +686,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 5, - "x": 5, - "y": 5 + "x": 6, + "y": 5, + "w": 6, + "h": 4 }, - "id": 247, + "id": 387, "options": { "colorMode": "value", "graphMode": "area", @@ -782,6 +834,172 @@ "title": "Database Resources", "type": "stat" }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data file size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 4, + "h": 4 + }, + "id": 388, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Storage Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "sishort" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 5, + "w": 4, + "h": 4 + }, + "id": 389, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Rows", + "type": "stat" + }, { "datasource": { "type": "mysql", @@ -807,12 +1025,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 5, - "x": 10, - "y": 5 + "x": 20, + "y": 5, + "w": 4, + "h": 4 }, - "id": 278, + "id": 390, "options": { "colorMode": "value", "graphMode": "area", @@ -926,15 +1144,449 @@ "title": "Data Size", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 9, + "w": 12, + "h": 6 + }, + "id": 538, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "ingestion", + "range": true, + "refId": "A" + } + ], + "title": "Total Ingestion Rate Trend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 9, + "w": 12, + "h": 6 + }, + "id": 539, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "queries", + "range": true, + "refId": "A" + } + ], + "title": "Total Query Rate Trend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 6 + }, + "id": 392, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "http-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-avg", + "range": true, + "refId": "B" + } + ], + "title": "HTTP Request P99 and Avg", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Tail and average latency for gRPC requests served by frontends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 6 + }, + "id": 557, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "grpc-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-avg", + "range": true, + "refId": "B" + } + ], + "title": "gRPC Request P99 and Avg", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 21 }, - "id": 275, + "id": 419, "panels": [], "title": "Ingestion", "type": "row" @@ -1001,12 +1653,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 24, "x": 0, - "y": 10 + "y": 22, + "w": 8, + "h": 6 }, - "id": 193, + "id": 413, "options": { "legend": { "calcs": [ @@ -1046,7 +1698,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "description": "Rows, samples, or documents ingested by primary observability and table-ingestion protocols.", "fieldConfig": { "defaults": { "color": { @@ -1103,12 +1755,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 16 + "x": 8, + "y": 22, + "w": 8, + "h": 6 }, - "id": 277, + "id": 414, "options": { "legend": { "calcs": [ @@ -1132,12 +1784,12 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "http-logs", + "legendFormat": "table-operator", "range": true, - "refId": "http_logs" + "refId": "A" }, { "datasource": { @@ -1145,15 +1797,4695 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", + "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~\"$frontend\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "prometheus-remote-write", "range": true, - "refId": "prometheus-remote-write" + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "loki-logs", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "elasticsearch-docs", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_metrics_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-metrics", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_logs_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-logs", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_traces_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-traces", + "range": true, + "refId": "H" } ], - "title": "Ingestion Rate by Type", + "title": "Ingestion Rate by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 22, + "w": 8, + "h": 6 + }, + "id": 415, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "prometheus-write", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "loki-logs", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "elasticsearch", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "otlp-metrics", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "otlp-logs", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "otlp-traces", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "prom-write-avg", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs-avg", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "loki-logs-avg", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "elasticsearch-avg", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-metrics-avg", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-logs-avg", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-traces-avg", + "range": true, + "refId": "N" + } + ], + "title": "Ingestion Latency by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bytes" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 8, + "h": 6 + }, + "id": 416, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "rows", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "bytes", + "range": true, + "refId": "B" + } + ], + "title": "Bulk Insert Message Rows and Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Remote-write pending-row flush operations, flushed rows, and p99 flush latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 28, + "w": 8, + "h": 6 + }, + "id": 417, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_prom_store_flush_total{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-ops", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_prom_store_flush_rows_sum{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-rows", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "flush-p99", + "range": true, + "refId": "C" + } + ], + "title": "Prom Store Flush Pipeline", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "OTLP trace ingestion failures reported by frontends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 28, + "w": 8, + "h": 6 + }, + "id": 418, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trace-failures", + "range": true, + "refId": "A" + } + ], + "title": "OTLP Trace Failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 403, + "panels": [], + "title": "Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 35, + "w": 8, + "h": 6 + }, + "id": 394, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (protocol) (rate(greptime_servers_error{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "server-{{protocol}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(greptime_servers_auth_failure_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "auth-{{code}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\",code!~\"2..\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-{{method}}-{{path}}-{{code}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\",code!~\"0|OK\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-{{path}}-{{code}}", + "range": true, + "refId": "D" + } + ], + "title": "Protocol Error Rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 35, + "w": 8, + "h": 6 + }, + "id": 395, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_request_memory_rejected_total{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "request-memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "query-memory", + "range": true, + "refId": "B" + } + ], + "title": "Frontend and Query Rejections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 35, + "w": 8, + "h": 6 + }, + "id": 396, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "region-request-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Datanode Write Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 8, + "h": 6 + }, + "id": 397, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_pending_rows_flush_failures{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-failures", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "dropped-rows", + "range": true, + "refId": "B" + } + ], + "title": "Buffered Ingestion Loss", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 41, + "w": 8, + "h": 6 + }, + "id": 398, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_write_reject_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "write-reject", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_write_stall_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "write-stall", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_flush_failure_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-failure", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_compaction_failure_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "compaction-failure", + "range": true, + "refId": "D" + } + ], + "title": "Mito Backpressure and Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Datanode scan and compaction memory rejection/exhaustion counters.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 41, + "w": 8, + "h": 6 + }, + "id": 399, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_scan_requests_rejected_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "scan-rejected", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "scan-exhausted", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "compaction-rejected", + "range": true, + "refId": "C" + } + ], + "title": "Scan and Compaction Memory Rejects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Object-store errors by scheme, operation, and error, excluding NotFound noise.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 47, + "w": 8, + "h": 6 + }, + "id": 400, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\",error!=\"NotFound\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{scheme}}-{{operation}}-{{error}}", + "range": true, + "refId": "A" + } + ], + "title": "OpenDAL Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region migration and reconciliation failures in metasrv.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 47, + "w": 8, + "h": 6 + }, + "id": 401, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "migration-fail", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "reconciliation-error", + "range": true, + "refId": "B" + } + ], + "title": "Metasrv Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Derived-data and alerting pipeline failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 47, + "w": 8, + "h": 6 + }, + "id": 402, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(greptime_flow_errors[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flow-{{code}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trigger-evaluate", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trigger-send-alert", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trigger-save-alert", + "range": true, + "refId": "D" + } + ], + "title": "Flow and Trigger Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito garbage-collection errors and skipped/orphaned files on datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 53, + "w": 8, + "h": 6 + }, + "id": 550, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_errors_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "gc-errors", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_orphaned_index_files{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "orphaned-index-files", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "skipped-unparsable", + "range": true, + "refId": "C" + } + ], + "title": "Mito GC Failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 411, + "panels": [], + "title": "Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 60, + "w": 8, + "h": 6 + }, + "id": 404, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_runtime_threads_alive)", + "hide": false, + "instant": false, + "legendFormat": "alive-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_runtime_threads_idle)", + "hide": false, + "instant": false, + "legendFormat": "idle-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Runtime Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Frontend request memory usage divided by configured request memory limit.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 60, + "w": 8, + "h": 6 + }, + "id": 405, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~\"$frontend\"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 60, + "w": 8, + "h": 6 + }, + "id": 406, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Query Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Datanode scan memory usage and compaction memory utilization.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 66, + "w": 8, + "h": 6 + }, + "id": 407, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "scan-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "compaction-used-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "compaction-limit-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + } + ], + "title": "Scan and Compaction Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 66, + "w": 8, + "h": 6 + }, + "id": 408, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "buffer-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "stalling-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Write Buffer and Active Stalling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 66, + "w": 8, + "h": 6 + }, + "id": 409, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "rows-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "batches-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "workers-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + } + ], + "title": "Prom Store Backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current in-flight flush and compaction tasks on datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 8, + "h": 6 + }, + "id": 410, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "flush-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "compaction-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Inflight Flush and Compaction", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 412, + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 465, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Frontend CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 463, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Datanode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 467, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$metasrv\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Metasrv CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 87 + }, + "id": 464, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Frontend Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 87 + }, + "id": 462, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Datanode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 87 + }, + "id": 466, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$metasrv\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Metasrv Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 95 + }, + "id": 469, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$flownode\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Flownode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 95 + }, + "id": 468, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$flownode\"})", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Flownode Memory per Instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 427, + "panels": [], + "title": "Queries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Query API call rates by protocol, collected from frontends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 104, + "w": 8, + "h": 6 + }, + "id": 420, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "mysql", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "postgres", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_sql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-sql", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-query", + "range": true, + "refId": "E" + } + ], + "title": "Query Rate by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "p95, p99, and average query latency by main frontend protocol.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 104, + "w": 8, + "h": 6 + }, + "id": 421, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "mysql-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "mysql-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "postgres-p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "postgres-p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "promql-p95", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "promql-p99", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "mysql-avg", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "postgres-avg", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql-avg", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-query-avg", + "range": true, + "refId": "J" + } + ], + "title": "Query Latency by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 104, + "w": 8, + "h": 6 + }, + "id": 422, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p95-{{stage}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99-{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Query Stage Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 110, + "w": 8, + "h": 6 + }, + "id": 423, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_query_merge_scan_regions)", + "hide": false, + "instant": false, + "legendFormat": "regions-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "errors-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Merge Scan Fan-out and Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 110, + "w": 8, + "h": 6 + }, + "id": 424, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "pushdown-fallback-errors", + "range": true, + "refId": "A" + } + ], + "title": "Pushdown Fallback Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 110, + "w": 8, + "h": 6 + }, + "id": 425, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_promql_series_count)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "PromQL Series Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 116, + "w": 8, + "h": 6 + }, + "id": 426, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "mysql-connections-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "postgres-connections-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "mysql-prepared-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "postgres-prepared-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "D" + } + ], + "title": "Connections and Prepared Statements", "type": "timeseries" }, { @@ -1162,1092 +6494,9 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 122 }, - "id": 276, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 246 - }, - "id": 255, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "mysql", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "pg", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~\"$frontend\"}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "promql", - "range": true, - "refId": "C" - } - ], - "title": "Total Query Rate", - "type": "timeseries" - } - ], - "title": "Queries", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 274, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 247 - }, - "id": 256, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$datanode\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Datanode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 247 - }, - "id": 262, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$datanode\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Datanode CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 257 - }, - "id": 266, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$frontend\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Frontend Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 257 - }, - "id": 268, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$frontend\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Frontend CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 267 - }, - "id": 269, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$metasrv\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Metasrv Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 267 - }, - "id": 271, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$metasrv\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Metasrv CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 277 - }, - "id": 272, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$flownode\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Flownode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 277 - }, - "id": 273, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$flownode\"})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Flownode CPU Usage per Instance", - "type": "timeseries" - } - ], - "title": "Resources", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 280, + "id": 428, "panels": [ { "datasource": { @@ -2312,11 +6561,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 288 + "y": 123 }, - "id": 281, + "id": 470, "options": { "legend": { "calcs": [ @@ -2355,7 +6604,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "HTTP P99 per Instance.", + "description": "HTTP P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -2413,11 +6662,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 288 + "w": 8, + "x": 8, + "y": 123 }, - "id": 282, + "id": 471, "options": { "legend": { "calcs": [ @@ -2446,9 +6695,22 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg", + "range": true, + "refId": "B" } ], - "title": "HTTP P99 per Instance", + "title": "HTTP P99 and Avg per Instance", "type": "timeseries" }, { @@ -2514,11 +6776,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 296 + "w": 8, + "x": 16, + "y": 123 }, - "id": 283, + "id": 472, "options": { "legend": { "calcs": [ @@ -2557,7 +6819,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "gRPC P99 per Instance.", + "description": "gRPC P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -2615,11 +6877,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 296 + "w": 8, + "x": 0, + "y": 131 }, - "id": 284, + "id": 473, "options": { "legend": { "calcs": [], @@ -2646,9 +6908,22 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg", + "range": true, + "refId": "B" } ], - "title": "gRPC P99 per Instance", + "title": "gRPC P99 and Avg per Instance", "type": "timeseries" }, { @@ -2714,11 +6989,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 304 + "w": 8, + "x": 8, + "y": 131 }, - "id": 285, + "id": 474, "options": { "legend": { "calcs": [ @@ -2757,7 +7032,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "MySQL P99 per Instance.", + "description": "MySQL P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -2815,11 +7090,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 304 + "w": 8, + "x": 16, + "y": 131 }, - "id": 286, + "id": 475, "options": { "legend": { "calcs": [ @@ -2849,9 +7124,22 @@ "legendFormat": "[{{ instance }}]-[{{ pod }}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "B" } ], - "title": "MySQL P99 per Instance", + "title": "MySQL P99 and Avg per Instance", "type": "timeseries" }, { @@ -2917,11 +7205,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 312 + "y": 139 }, - "id": 287, + "id": 476, "options": { "legend": { "calcs": [ @@ -2960,7 +7248,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "PostgreSQL P99 per Instance.", + "description": "PostgreSQL P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -3018,11 +7306,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 312 + "w": 8, + "x": 8, + "y": 139 }, - "id": 288, + "id": 477, "options": { "legend": { "calcs": [], @@ -3049,9 +7337,22 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "B" } ], - "title": "PostgreSQL P99 per Instance", + "title": "PostgreSQL P99 and Avg per Instance", "type": "timeseries" } ], @@ -3064,111 +7365,10 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 123 }, - "id": 289, + "id": 429, "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ingestion rate by row as in each frontend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 289 - }, - "id": 292, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Ingest Rows per Instance", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -3232,11 +7432,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 295 + "y": 124 }, - "id": 290, + "id": 479, "options": { "legend": { "calcs": [ @@ -3275,7 +7475,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Region Call P99 per Instance.", + "description": "Region Call P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -3333,11 +7533,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 295 + "w": 8, + "x": 8, + "y": 124 }, - "id": 291, + "id": 480, "options": { "legend": { "calcs": [ @@ -3366,9 +7566,22 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg", + "range": true, + "refId": "B" } ], - "title": "Region Call P99 per Instance", + "title": "Region Call P99 and Avg per Instance", "type": "timeseries" }, { @@ -3430,43 +7643,15 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[minipc-1:4000]-[]-[encode]-AVG", - "[minipc-1:4000]-[]-[partition]-AVG", - "[minipc-1:4000]-[]-[decode_request]-AVG", - "[minipc-1:4000]-[]-[datanode_handle]-AVG" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 303 + "w": 8, + "x": 16, + "y": 124 }, - "id": 336, + "id": 481, "options": { "legend": { "calcs": [ @@ -3510,22 +7695,662 @@ "refId": "B" } ], - "title": "Frontend Handle Bulk Insert Elapsed Time ", + "title": "Frontend Handle Bulk Insert Elapsed Time", "type": "timeseries" } ], "title": "Frontend to Datanode", "type": "row" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 124 + }, + "id": 434, + "panels": [], + "title": "Datanode", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Datanode region request failures and failed inserts by instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 125, + "w": 8, + "h": 6 + }, + "id": 430, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "request-fail-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Region Request Failures and Failed Inserts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito write rejects, write stall events, and active write stalling by datanode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 125, + "w": 8, + "h": 6 + }, + "id": 431, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "reject-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "stall-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "active-stalling-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + } + ], + "title": "Write Rejects and Stalls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito flush and compaction failure rates by datanode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 125, + "w": 8, + "h": 6 + }, + "id": 432, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "compaction-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Flush and Compaction Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 131, + "w": 8, + "h": 6 + }, + "id": 433, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_runs_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "runs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_errors_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "errors", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_files_deleted_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "files-deleted", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_orphaned_index_files{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "orphaned-index-files", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "skipped-unparsable", + "range": true, + "refId": "E" + } + ], + "title": "Mito GC Health", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "P99 and average Mito garbage-collection duration by stage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 131, + "w": 8, + "h": 6 + }, + "id": 551, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}-avg", + "range": true, + "refId": "B" + } + ], + "title": "Mito GC Duration", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 137 }, - "id": 293, + "id": 435, "panels": [ { "datasource": { @@ -3590,11 +8415,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 193 + "y": 138 }, - "id": 294, + "id": 482, "options": { "legend": { "calcs": [ @@ -3633,7 +8458,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Request P99 per Instance.", + "description": "Request P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -3691,11 +8516,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 193 + "w": 8, + "x": 8, + "y": 138 }, - "id": 295, + "id": 483, "options": { "legend": { "calcs": [ @@ -3724,9 +8549,150 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg", + "range": true, + "refId": "B" } ], - "title": "Request P99 per Instance", + "title": "Request P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 138 + }, + "id": 556, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-avg", + "range": true, + "refId": "C" + } + ], + "title": "Request Wait P99 and Avg per Worker", "type": "timeseries" }, { @@ -3792,11 +8758,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 225 + "y": 146 }, - "id": 296, + "id": 484, "options": { "legend": { "calcs": [ @@ -3893,11 +8859,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 225 + "w": 8, + "x": 8, + "y": 146 }, - "id": 297, + "id": 485, "options": { "legend": { "calcs": [ @@ -3931,207 +8897,6 @@ "title": "Write Rows per Instance", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Flush QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 233 - }, - "id": 298, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", - "range": true, - "refId": "A" - } - ], - "title": "Flush OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stall per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 233 - }, - "id": 299, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Stall per Instance", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -4195,11 +8960,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 241 + "w": 8, + "x": 16, + "y": 146 }, - "id": 300, + "id": 488, "options": { "legend": { "calcs": [ @@ -4238,133 +9003,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Read Stage P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 241 - }, - "id": 301, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stage P99 per Instance.", + "description": "Read Stage P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -4422,11 +9061,125 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 249 + "y": 154 }, - "id": 302, + "id": 489, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Read Stage P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stage P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 154 + }, + "id": 490, "options": { "legend": { "calcs": [ @@ -4457,208 +9210,6 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", "range": true, "refId": "A" - } - ], - "title": "Write Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction OPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 249 - }, - "id": 303, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Compaction OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction latency by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 257 - }, - "id": 304, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", - "range": true, - "refId": "A" }, { "datasource": { @@ -4666,7 +9217,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", @@ -4674,235 +9225,7 @@ "refId": "B" } ], - "title": "Compaction Elapsed Time per Instance by Stage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 257 - }, - "id": 305, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", - "range": true, - "refId": "A" - } - ], - "title": "Compaction P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 265 - }, - "id": 306, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", - "range": true, - "refId": "C" - } - ], - "title": "WAL write size", + "title": "Write Stage P99 and Avg per Instance", "type": "timeseries" }, { @@ -4964,40 +9287,15 @@ }, "unit": "decbytes" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 265 + "w": 8, + "x": 16, + "y": 154 }, - "id": 307, + "id": 495, "options": { "legend": { "calcs": [], @@ -5029,524 +9327,6 @@ "title": "Cached Bytes per Instance", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing compaction task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 273 - }, - "id": 308, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_compaction_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Compaction", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Raft engine (local disk) log store sync latency, p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 273 - }, - "id": 310, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "WAL sync duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead log operations latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 281 - }, - "id": 311, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Log Store op duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing flush task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 281 - }, - "id": 312, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_flush_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Flush", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction oinput output bytes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 289 - }, - "id": 335, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-input", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-output", - "range": true, - "refId": "B" - } - ], - "title": "Compaction Input/Output Bytes", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -5606,44 +9386,15 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[bulk_extend]-AVG", - "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[process_bulk_req]-AVG", - "[minipc-1:4000]-[]-[write_bulk]-AVG" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 289 + "w": 8, + "x": 0, + "y": 162 }, - "id": 337, + "id": 501, "options": { "legend": { "calcs": [ @@ -5695,7 +9446,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Compaction oinput output bytes", + "description": "Active series and field-builder counts per memtable by instance.", "fieldConfig": { "defaults": { "color": { @@ -5753,11 +9504,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 297 + "w": 8, + "x": 8, + "y": 162 }, - "id": 348, + "id": 502, "options": { "legend": { "calcs": [ @@ -5867,11 +9618,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 297 + "w": 8, + "x": 16, + "y": 162 }, - "id": 338, + "id": 503, "options": { "legend": { "calcs": [ @@ -5984,11 +9735,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 305 + "y": 170 }, - "id": 349, + "id": 504, "options": { "legend": { "calcs": [ @@ -6022,7 +9773,7 @@ "type": "timeseries" } ], - "title": "Mito Engine", + "title": "Storage", "type": "row" }, { @@ -6031,16 +9782,16 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 138 }, - "id": 313, + "id": 555, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "QPS per Instance.", + "description": "Flush QPS per Instance.", "fieldConfig": { "defaults": { "color": { @@ -6053,6 +9804,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6096,12 +9848,12 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 8, "x": 0, - "y": 459 + "y": 139 }, - "id": 314, + "id": 486, "options": { "legend": { "calcs": [ @@ -6117,7 +9869,7 @@ "sort": "desc" } }, - "pluginVersion": "12.0.0", + "pluginVersion": "11.6.0", "targets": [ { "datasource": { @@ -6125,14 +9877,14 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", + "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", "range": true, "refId": "A" } ], - "title": "QPS per Instance", + "title": "Flush OPS per Instance", "type": "timeseries" }, { @@ -6140,7 +9892,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Read QPS per Instance.", + "description": "Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs.", "fieldConfig": { "defaults": { "color": { @@ -6153,6 +9905,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6191,823 +9944,21 @@ } ] }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 469 - }, - "id": 315, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 469 + "h": 8, + "w": 8, + "x": 8, + "y": 139 }, - "id": 316, + "id": 540, "options": { "legend": { "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=~\"read|Reader::read\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 476 - }, - "id": 317, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 476 - }, - "id": 318, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 483 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 483 - }, - "id": 320, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Requests per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 490 - }, - "id": 321, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Requests per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Request P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 490 - }, - "id": 322, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|write|list|Writer::write|Writer::close|Reader::read\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Total traffic as in bytes by instance and operation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 497 - }, - "id": 323, - "options": { - "legend": { - "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -7019,26 +9970,49 @@ "sort": "none" } }, - "pluginVersion": "12.0.0", + "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg", + "range": true, + "refId": "C" } ], - "title": "Opendal traffic", + "title": "Flush Elapsed Time", "type": "timeseries" }, { @@ -7046,7 +10020,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "OpenDAL error counts per Instance.", + "description": "Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs.", "fieldConfig": { "defaults": { "color": { @@ -7059,6 +10033,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -7096,17 +10071,247 @@ "value": 80 } ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*-files$" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 139 + }, + "id": 541, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-bytes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-files", + "range": true, + "refId": "B" + } + ], + "title": "Flush Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing flush task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 497 + "h": 8, + "w": 8, + "x": 0, + "y": 147 }, - "id": 334, + "id": 499, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_flush_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Flush", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 147 + }, + "id": 491, "options": { "legend": { "calcs": [ @@ -7122,7 +10327,7 @@ "sort": "desc" } }, - "pluginVersion": "12.0.0", + "pluginVersion": "11.6.0", "targets": [ { "datasource": { @@ -7130,18 +10335,461 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\", error!=\"NotFound\"}[$__rate_interval]))", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", + "legendFormat": "[{{ instance }}]-[{{pod}}]", "range": true, "refId": "A" } ], - "title": "OpenDAL errors per Instance", + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing compaction task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 147 + }, + "id": 496, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_compaction_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Compaction", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 155 + }, + "id": 493, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Compaction P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction latency by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 155 + }, + "id": 492, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Elapsed Time per Instance by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 155 + }, + "id": 500, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-input", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-output", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Input/Output Bytes", "type": "timeseries" } ], - "title": "OpenDAL", + "title": "Flush and Compaction", "type": "row" }, { @@ -7150,16 +10798,16 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 139 }, - "id": 351, + "id": 549, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "Triggered region flush total", + "description": "Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates.", "fieldConfig": { "defaults": { "color": { @@ -7211,21 +10859,23 @@ } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 195 + "y": 164, + "w": 8, + "h": 6 }, - "id": 350, + "id": 542, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7243,14 +10893,28 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "meta_triggered_region_flush_total", + "expr": "histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", + "legendFormat": "{{type}}-p95", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-p99", + "range": true, + "refId": "B" } ], - "title": "Triggered region flush total", + "title": "Index Apply Elapsed Time", "type": "timeseries" }, { @@ -7258,7 +10922,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Triggered region checkpoint total", + "description": "Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays.", "fieldConfig": { "defaults": { "color": { @@ -7310,21 +10974,23 @@ } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 195 + "x": 8, + "y": 164, + "w": 8, + "h": 6 }, - "id": 352, + "id": 543, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7342,14 +11008,28 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "meta_triggered_region_checkpoint_total", + "expr": "histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", + "legendFormat": "{{type}}-{{stage}}-p95", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-{{stage}}-p99", + "range": true, + "refId": "B" } ], - "title": "Triggered region checkpoint total", + "title": "Index Create Elapsed Time", "type": "timeseries" }, { @@ -7357,7 +11037,135 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Topic estimated max replay size", + "description": "Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*-bytes$" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + }, + "gridPos": { + "x": 16, + "y": 164, + "w": 8, + "h": 6 + }, + "id": 544, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-rows", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-bytes", + "range": true, + "refId": "B" + } + ], + "title": "Index Create Rows and Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work.", "fieldConfig": { "defaults": { "color": { @@ -7414,16 +11222,18 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 227 + "y": 170, + "w": 8, + "h": 6 }, - "id": 353, + "id": 545, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7441,14 +11251,28 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "meta_topic_estimated_replay_size", + "expr": "greptime_index_apply_memory_usage", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", + "legendFormat": "apply", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (greptime_index_create_memory_usage)", + "hide": false, + "instant": false, + "legendFormat": "create-{{type}}", + "range": true, + "refId": "B" } ], - "title": "Topic estimated replay size", + "title": "Index Memory Usage", "type": "timeseries" }, { @@ -7456,7 +11280,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Kafka logstore's bytes traffic", + "description": "Index read and write byte rates by operation and file type for puffin and intermediate files.", "fieldConfig": { "defaults": { "color": { @@ -7508,21 +11332,23 @@ } ] }, - "unit": "bytes" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 227 + "x": 8, + "y": 170, + "w": 8, + "h": 6 }, - "id": 354, + "id": 546, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7540,18 +11366,249 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", + "expr": "sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{logstore}}", + "legendFormat": "{{type}}-{{file_type}}", "range": true, "refId": "A" } ], - "title": "Kafka logstore's bytes traffic", + "title": "Index IO Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Index IO operation rates by operation and file type, including read, write, seek, and flush operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 170, + "w": 8, + "h": 6 + }, + "id": 547, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-{{file_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Index IO Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Index-related cache hits, misses, and evictions from Mito caches.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 176, + "w": 8, + "h": 6 + }, + "id": 548, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "hit-{{type}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "miss-{{type}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "evict-{{type}}-{{cause}}", + "range": true, + "refId": "C" + } + ], + "title": "Index Cache", "type": "timeseries" } ], - "title": "Remote WAL", + "title": "Index", "type": "row" }, { @@ -7560,10 +11617,292 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 140 }, - "id": 324, + "id": 451, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 165, + "w": 8, + "h": 8 + }, + "id": 436, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(greptime_meta_inactive_regions)", + "hide": false, + "instant": false, + "legendFormat": "inactive-regions", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(greptime_lease_expired_region{instance=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "lease-expired", + "range": true, + "refId": "B" + } + ], + "title": "Inactive and Lease-expired Regions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 165, + "w": 8, + "h": 8 + }, + "id": 437, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "meta-recv-rate", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(greptime_meta_heartbeat_connection_num)", + "hide": false, + "instant": false, + "legendFormat": "meta-connections", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_heartbeat_send_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "frontend-send", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_heartbeat_recv_count{instance=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "frontend-recv", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_datanode_heartbeat_send_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "datanode-send", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_datanode_heartbeat_recv_count{instance=~\"$datanode\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "datanode-recv", + "range": true, + "refId": "F" + } + ], + "title": "Heartbeat Health", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -7602,12 +11941,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 196 + "x": 16, + "y": 165, + "w": 8, + "h": 8 }, - "id": 325, + "id": 525, "options": { "colWidth": 0.9, "legend": { @@ -7716,12 +12055,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 196 + "x": 0, + "y": 173, + "w": 8, + "h": 8 }, - "id": 326, + "id": 526, "options": { "legend": { "calcs": [], @@ -7743,7 +12082,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "greptime_meta_region_migration_error", + "expr": "rate(greptime_meta_region_migration_error[$__rate_interval])", "instant": false, "legendFormat": "{{pod}}-{{state}}-{{error_type}}", "range": true, @@ -7815,12 +12154,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 228 + "x": 8, + "y": 173, + "w": 8, + "h": 8 }, - "id": 327, + "id": 527, "options": { "legend": { "calcs": [], @@ -7914,12 +12253,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 228 + "x": 16, + "y": 173, + "w": 8, + "h": 8 }, - "id": 339, + "id": 528, "options": { "legend": { "calcs": [], @@ -8013,12 +12352,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 236 + "y": 181, + "w": 8, + "h": 8 }, - "id": 340, + "id": 529, "options": { "legend": { "calcs": [], @@ -8049,6 +12388,19 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg", + "range": true, + "refId": "B" } ], "title": "SQL Execution Latency (RDS)", @@ -8116,12 +12468,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 236 + "x": 8, + "y": 181, + "w": 8, + "h": 8 }, - "id": 341, + "id": 530, "options": { "legend": { "calcs": [], @@ -8152,6 +12504,19 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{pod}}]-[{{name}}]-avg", + "range": true, + "refId": "B" } ], "title": "Handler Execution Latency", @@ -8219,12 +12584,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 244 + "x": 16, + "y": 181, + "w": 8, + "h": 8 }, - "id": 342, + "id": 531, "options": { "legend": { "calcs": [], @@ -8247,7 +12612,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", + "expr": "histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -8265,7 +12630,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "Rate of heartbeats received by metasrv from datanodes and frontends.", "fieldConfig": { "defaults": { "color": { @@ -8322,12 +12687,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 244 + "x": 0, + "y": 189, + "w": 8, + "h": 8 }, - "id": 345, + "id": 532, "options": { "legend": { "calcs": [], @@ -8368,7 +12733,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "p99 and average latency of metasrv key-value store operations by op and target.", "fieldConfig": { "defaults": { "color": { @@ -8425,12 +12790,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 252 + "x": 8, + "y": 189, + "w": 8, + "h": 8 }, - "id": 343, + "id": 533, "options": { "legend": { "calcs": [], @@ -8453,7 +12818,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", + "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -8461,6 +12826,19 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{pod}}]-[{{op}}]-[{{target}}]-avg", + "range": true, + "refId": "B" } ], "title": "Meta KV Ops Latency", @@ -8471,7 +12849,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "Rate of metasrv key-value store operations by op.", "fieldConfig": { "defaults": { "color": { @@ -8528,12 +12906,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 252 + "x": 16, + "y": 189, + "w": 8, + "h": 8 }, - "id": 346, + "id": 534, "options": { "legend": { "calcs": [], @@ -8574,7 +12952,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step.", "fieldConfig": { "defaults": { "color": { @@ -8631,12 +13009,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 260 + "y": 197, + "w": 8, + "h": 8 }, - "id": 347, + "id": 535, "options": { "legend": { "calcs": [], @@ -8659,7 +13037,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -8675,7 +13053,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8692,7 +13070,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8709,7 +13087,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8726,7 +13104,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8743,7 +13121,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8814,17 +13192,17 @@ } ] }, - "unit": "s" + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 260 + "x": 8, + "y": 197, + "w": 8, + "h": 8 }, - "id": 355, + "id": 536, "options": { "legend": { "calcs": [], @@ -8846,7 +13224,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "greptime_meta_reconciliation_stats", + "expr": "rate(greptime_meta_reconciliation_stats[$__rate_interval])", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{table_type}}-{{type}}", @@ -8919,12 +13297,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 268 + "x": 16, + "y": 197, + "w": 8, + "h": 8 }, - "id": 356, + "id": 537, "options": { "legend": { "calcs": [], @@ -8946,7 +13324,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", + "expr": "histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{procedure_name}}-{{step}}-P90", @@ -8967,9 +13345,3174 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 141 }, - "id": 328, + "id": 553, + "panels": [ + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": false + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "disk_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 166, + "w": 24, + "h": 18 + }, + "id": 438, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [ + "region_rows", + "disk_size" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Hotspot Regions", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write load of each datanode over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 184, + "w": 8, + "h": 11 + }, + "id": 439, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "greptime_datanode_history_load", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Distribution of write load across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 184, + "w": 8, + "h": 11 + }, + "id": 440, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "greptime_datanode_history_load", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write) Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Distribution of leader regions and data size across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 184, + "w": 8, + "h": 11 + }, + "id": 441, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^data_size$/", + "values": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Datanode Data Distribution", + "type": "piechart" + } + ], + "title": "Hotspot", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 554, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer action count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 167, + "w": 8, + "h": 8 + }, + "id": 442, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Actions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer gate stop count by gate and reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 167, + "w": 8, + "h": 8 + }, + "id": 443, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{gate}} / {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Gate Stops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer datanode count by state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 167, + "w": 8, + "h": 8 + }, + "id": 444, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (state) (greptime_region_balancer_datanodes)", + "hide": false, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Datanodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer region count by state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 175, + "w": 8, + "h": 8 + }, + "id": 445, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (state) (greptime_region_balancer_regions)", + "hide": false, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Regions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer datanode stability statistics by state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*count$" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*ratio$" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "x": 8, + "y": 175, + "w": 8, + "h": 8 + }, + "id": 446, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "builder", + "expr": "sum by (state) (greptime_region_balancer_datanode_stability)", + "hide": false, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Datanode Stability", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition action count by result", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": -1, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 175, + "w": 8, + "h": 8 + }, + "id": 447, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Actions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition gate stop count by gate and reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 183, + "w": 8, + "h": 8 + }, + "id": 448, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{gate}} / {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Gate Stops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition sampling elapsed time by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 183, + "w": 8, + "h": 8 + }, + "id": 449, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Sampling P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition executor elapsed time by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 183, + "w": 8, + "h": 8 + }, + "id": 450, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Executor P99", + "type": "timeseries" + } + ], + "title": "Autopilot", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 143 + }, + "id": 452, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 168 + }, + "id": 505, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 178 + }, + "id": 506, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 178 + }, + "id": 507, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=~\"read|Reader::read\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Read P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 178 + }, + "id": 508, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 185 + }, + "id": 509, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Write P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 185 + }, + "id": 510, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 185 + }, + "id": 511, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "List P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Requests per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 192 + }, + "id": 512, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Request P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 192 + }, + "id": 513, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Other Request P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total traffic as in bytes by instance and operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 192 + }, + "id": 514, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Opendal traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "OpenDAL error counts per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 199 + }, + "id": 515, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\", error!=\"NotFound\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", + "range": true, + "refId": "A" + } + ], + "title": "OpenDAL errors per Instance", + "type": "timeseries" + } + ], + "title": "Object Store", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 144 + }, + "id": 552, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 169 + }, + "id": 494, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", + "range": true, + "refId": "C" + } + ], + "title": "WAL write size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Raft engine (local disk) log store sync latency, p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 169 + }, + "id": 497, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "WAL sync duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead log operations latency at p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 169 + }, + "id": 498, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Log Store op duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region flush total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 177 + }, + "id": 516, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_flush_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region flush total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region checkpoint total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 177 + }, + "id": 517, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_checkpoint_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region checkpoint total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Topic estimated max replay size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 177 + }, + "id": 518, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_topic_estimated_replay_size", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Topic estimated replay size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Kafka logstore's bytes traffic", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 185 + }, + "id": 519, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", + "instant": false, + "legendFormat": "{{pod}}-{{logstore}}", + "range": true, + "refId": "A" + } + ], + "title": "Kafka logstore's bytes traffic", + "type": "timeseries" + } + ], + "title": "WAL", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 145 + }, + "id": 453, "panels": [ { "datasource": { @@ -9032,11 +16575,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 3282 + "y": 170 }, - "id": 329, + "id": 520, "options": { "legend": { "calcs": [], @@ -9127,11 +16670,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 3282 + "w": 8, + "x": 8, + "y": 170 }, - "id": 330, + "id": 521, "options": { "legend": { "calcs": [], @@ -9169,6 +16712,19 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "C" } ], "title": "Flow Ingest Latency", @@ -9235,11 +16791,11 @@ }, "gridPos": { "h": 8, - "w": 9, - "x": 0, - "y": 3290 + "w": 8, + "x": 16, + "y": 170 }, - "id": 331, + "id": 522, "options": { "legend": { "calcs": [], @@ -9343,11 +16899,11 @@ }, "gridPos": { "h": 8, - "w": 9, - "x": 9, - "y": 3290 + "w": 8, + "x": 0, + "y": 178 }, - "id": 332, + "id": 523, "options": { "legend": { "calcs": [], @@ -9438,11 +16994,11 @@ }, "gridPos": { "h": 8, - "w": 6, - "x": 18, - "y": 3290 + "w": 8, + "x": 8, + "y": 178 }, - "id": 333, + "id": 524, "options": { "legend": { "calcs": [], @@ -9477,2077 +17033,793 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 146 }, - "id": 357, - "panels": [], - "title": "Trigger", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "bf9fzta69bhtsa" - }, - "description": "Total number of triggers currently defined.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 461, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "description": "Total number of triggers currently defined.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 358, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_trigger_count{}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Trigger Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 359, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p75", - "range": true, - "refId": "B" - } - ], - "title": "Trigger Eval Elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Rate of failed trigger evaluations.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 360, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Trigger Eval Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed time to send trigger alerts to notification channels.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 361, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75", - "range": true, - "refId": "B" - } - ], - "title": "Send Alert Elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Rate of failures when sending trigger alerts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 364, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Send Alert Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed time to persist trigger alert records.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 363, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75", - "range": true, - "refId": "B" - } - ], - "title": "Save Alert Elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Rate of failures when persisting trigger alert records.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 362, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Save Alert Failure Rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 366, - "panels": [], - "title": "Hotspot", - "type": "row" - }, - { - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": false - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "disk_size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 18, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 365, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": [ - "region_rows", - "disk_size" - ], - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Hotspot Regions", - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write load of each datanode over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "total_region_rows", - "total_written_bytes_since_open", - "total_disk_size", - "total_sst_size", - "total_sst_num", - "total_memtable_size", - "total_manifest_size", - "total_index_size", - "cluster_disk_ratio_percent", - "cluster_sst_ratio_percent", - "cluster_rows_ratio_percent", - "max_region_disk_size", - "max_region_rows", - "leader_region_count", - "disk_ratio_percent", - "data_size" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 368, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "greptime_datanode_history_load", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "datanode-{{datanode_id}}({{instance}})", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Datanode Load(Write)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Distribution of write load across datanodes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "fieldMinMax": false, - "mappings": [], - "min": 0, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 369, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "greptime_datanode_history_load", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "datanode-{{datanode_id}}({{instance}})", - "range": false, - "refId": "A", - "useBackend": false - } - ], - "title": "Datanode Load(Write) Distribution", - "type": "piechart" - }, - { - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "description": "Distribution of leader regions and data size across datanodes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "fieldMinMax": false, - "mappings": [], - "unit": "bytes" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "total_region_rows", - "total_written_bytes_since_open", - "total_disk_size", - "total_sst_size", - "total_sst_num", - "total_memtable_size", - "total_manifest_size", - "total_index_size", - "cluster_disk_ratio_percent", - "cluster_sst_ratio_percent", - "cluster_rows_ratio_percent", - "max_region_disk_size", - "max_region_rows", - "leader_region_count", - "disk_ratio_percent", - "data_size" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 94 - }, - "id": 367, - "options": { - "displayLabels": [ - "value" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^data_size$/", - "values": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" + "viz": false }, - "type": "groupBy" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } - ], - "limit": 50 - } + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 171, + "w": 8, + "h": 6 + }, + "id": 454, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_trigger_count{}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Trigger Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 171, + "w": 8, + "h": 6 + }, + "id": 455, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "C" + } + ], + "title": "Trigger Eval Elapsed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Rate of failed trigger evaluations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 171, + "w": 8, + "h": 6 + }, + "id": 456, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Trigger Eval Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed time to send trigger alerts to notification channels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 177, + "w": 8, + "h": 6 + }, + "id": 457, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "C" + } + ], + "title": "Send Alert Elapsed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Rate of failures when sending trigger alerts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 177, + "w": 8, + "h": 6 + }, + "id": 458, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Send Alert Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed time to persist trigger alert records.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 177, + "w": 8, + "h": 6 + }, + "id": 459, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "C" + } + ], + "title": "Save Alert Elapsed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Rate of failures when persisting trigger alert records.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 183, + "w": 8, + "h": 6 + }, + "id": 460, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Save Alert Failure Rate", + "type": "timeseries" } ], - "title": "Datanode Data Distribution", - "type": "piechart" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 105 - }, - "id": 370, - "panels": [], - "title": "Autopilot", + "title": "Trigger", "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer action count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 106 - }, - "id": 371, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "{{result}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Actions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer gate stop count by gate and reason", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 106 - }, - "id": 372, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{gate}} / {{reason}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Gate Stops", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer datanode count by state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 114 - }, - "id": 373, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (state) (greptime_region_balancer_datanodes)", - "hide": false, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Datanodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer region count by state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 114 - }, - "id": 374, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (state) (greptime_region_balancer_regions)", - "hide": false, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Regions", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer datanode stability statistics by state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "binBps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*count$" - }, - "properties": [ - { - "id": "unit", - "value": "short" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*ratio$" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 122 - }, - "id": 375, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "builder", - "expr": "sum by (state) (greptime_region_balancer_datanode_stability)", - "hide": false, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Datanode Stability", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition action count by result", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": -1, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 130 - }, - "id": 376, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{result}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Actions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition gate stop count by gate and reason", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 130 - }, - "id": 377, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{gate}} / {{reason}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Gate Stops", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition sampling elapsed time by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 138 - }, - "id": 378, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{stage}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Sampling P99", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition executor elapsed time by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 138 - }, - "id": 379, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{stage}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Executor P99", - "type": "timeseries" } ], "preload": false, @@ -11686,7 +17958,7 @@ }, "timepicker": {}, "timezone": "", - "title": "GreptimeDB Copy", + "title": "GreptimeDB", "uid": "dflfbxbwvvchsa", - "version": 67 -} \ No newline at end of file + "version": 68 +} diff --git a/grafana/dashboards/metrics/cluster/dashboard.md b/grafana/dashboards/metrics/cluster/dashboard.md index 2204c6d0a76..fd7ae68fd48 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.md +++ b/grafana/dashboards/metrics/cluster/dashboard.md @@ -4,144 +4,154 @@ | Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` | | Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- | | Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` | -| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- | -| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- | +| Total Query Rate | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `stat` | Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. | `prometheus` | `reqps` | `queries` | +| User-facing Error Rate | `sum(rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))` | `stat` | Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. | `prometheus` | `eps` | `errors` | +| Recent Restarts | `sum(changes(process_start_time_seconds[$__range]))` | `stat` | Process restarts over the selected time range across GreptimeDB roles. | `prometheus` | `short` | `restarts` | | Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- | | Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- | +| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- | +| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- | | Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- | +| Total Ingestion Rate Trend | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. | `prometheus` | `rowsps` | `ingestion` | +| Total Query Rate Trend | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. | `prometheus` | `reqps` | `queries` | +| HTTP Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))`
`sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. | `prometheus` | `s` | `http-p99` | +| gRPC Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Tail and average latency for gRPC requests served by frontends. | `prometheus` | `s` | `grpc-p99` | # Ingestion | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | | Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` | -| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `http-logs` | -# Queries +| Ingestion Rate by Protocol | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_metrics_rows{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_logs_rows{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_traces_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Rows, samples, or documents ingested by primary observability and table-ingestion protocols. | `prometheus` | `rowsps` | `table-operator` | +| Ingestion Latency by Protocol | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. | `prometheus` | `s` | `prometheus-write` | +| Bulk Insert Message Rows and Size | `sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. | `prometheus` | `rowsps` | `rows` | +| Prom Store Flush Pipeline | `sum(rate(greptime_prom_store_flush_total{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_prom_store_flush_rows_sum{instance=~"$frontend"}[$__rate_interval]))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Remote-write pending-row flush operations, flushed rows, and p99 flush latency. | `prometheus` | `short` | `flush-ops` | +| OTLP Trace Failures | `sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | OTLP trace ingestion failures reported by frontends. | `prometheus` | `eps` | `trace-failures` | +# Health | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` | +| Protocol Error Rates | `sum by (protocol) (rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))`
`sum by (code) (rate(greptime_servers_auth_failure_count{instance=~"$frontend"}[$__rate_interval]))`
`sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics",code!~"2.."}[$__rate_interval]))`
`sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend",code!~"0\|OK"}[$__rate_interval]))` | `timeseries` | User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. | `prometheus` | `eps` | `server-{{protocol}}` | +| Frontend and Query Rejections | `sum(rate(greptime_servers_request_memory_rejected_total{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))` | `timeseries` | Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. | `prometheus` | `rps` | `request-memory` | +| Datanode Write Failures | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. | `prometheus` | `eps` | `region-request-[{{instance}}]-[{{pod}}]` | +| Buffered Ingestion Loss | `sum(rate(greptime_pending_rows_flush_failures{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. | `prometheus` | `eps` | `flush-failures` | +| Mito Backpressure and Failures | `sum(rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. | `prometheus` | `eps` | `write-reject` | +| Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` | +| OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode",error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` | +| Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`
`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` | +| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`
`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` | +| Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` | +# Capacity +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Runtime Threads | `sum by (instance, pod) (greptime_runtime_threads_alive)`
`sum by (instance, pod) (greptime_runtime_threads_idle)` | `timeseries` | Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. | `prometheus` | `short` | `alive-[{{instance}}]-[{{pod}}]` | +| Request Memory Utilization | `sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~"$frontend"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~"$frontend"})` | `timeseries` | Frontend request memory usage divided by configured request memory limit. | `prometheus` | `percentunit` | `[{{instance}}]-[{{pod}}]` | +| Query Memory Usage | `sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)` | `timeseries` | Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]` | +| Scan and Compaction Memory | `sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~"$datanode"})` | `timeseries` | Datanode scan memory usage and compaction memory utilization. | `prometheus` | `bytes` | `scan-[{{instance}}]-[{{pod}}]` | +| Write Buffer and Active Stalling | `sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})` | `timeseries` | Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. | `prometheus` | `bytes` | `buffer-[{{instance}}]-[{{pod}}]` | +| Prom Store Backlog | `sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~"$frontend"})`
`sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~"$frontend"})`
`sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~"$frontend"})` | `timeseries` | Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. | `prometheus` | `short` | `rows-[{{instance}}]-[{{pod}}]` | +| Inflight Flush and Compaction | `sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~"$datanode"})` | `timeseries` | Current in-flight flush and compaction tasks on datanodes. | `prometheus` | `short` | `flush-[{{instance}}]-[{{pod}}]` | # Resources | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$datanode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` | -| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$datanode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$frontend"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | | Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$frontend"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` | -| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$metasrv"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` | +| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$datanode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | | Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$flownode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$frontend"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | +| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$datanode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` | +| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$metasrv"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` | | Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$flownode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$flownode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | +# Queries +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Query Rate by Protocol | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Query API call rates by protocol, collected from frontends. | `prometheus` | `reqps` | `mysql` | +| Query Latency by Protocol | `histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | p95, p99, and average query latency by main frontend protocol. | `prometheus` | `s` | `mysql-p95` | +| Query Stage Latency | `histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))` | `timeseries` | p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. | `prometheus` | `s` | `p95-{{stage}}` | +| Merge Scan Fan-out and Errors | `sum by (instance, pod) (greptime_query_merge_scan_regions)`
`sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))` | `timeseries` | Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. | `prometheus` | `short` | `regions-[{{instance}}]-[{{pod}}]` | +| Pushdown Fallback Errors | `sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))` | `timeseries` | Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. | `prometheus` | `eps` | `pushdown-fallback-errors` | +| PromQL Series Count | `sum by (instance, pod) (greptime_promql_series_count)` | `timeseries` | Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]` | +| Connections and Prepared Statements | `sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~"$frontend"})`
`sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~"$frontend"})`
`sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~"$frontend"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. | `prometheus` | `short` | `mysql-connections-[{{instance}}]-[{{pod}}]` | # Frontend Requests | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | | HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | -| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| HTTP P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))`
`sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | | gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | -| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| gRPC P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | | MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | -| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` | +| MySQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL P99 and average per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` | | PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | -| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| PostgreSQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | # Frontend to Datanode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | | Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | -| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | -| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` | -# Mito Engine +| Region Call P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` | +# Datanode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Region Request Failures and Failed Inserts | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode region request failures and failed inserts by instance. | `prometheus` | `eps` | `request-fail-[{{instance}}]-[{{pod}}]` | +| Write Rejects and Stalls | `sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})` | `timeseries` | Mito write rejects, write stall events, and active write stalling by datanode. | `prometheus` | `short` | `reject-[{{instance}}]-[{{pod}}]` | +| Flush and Compaction Failures | `sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush and compaction failure rates by datanode. | `prometheus` | `eps` | `flush-[{{instance}}]-[{{pod}}]` | +| Mito GC Health | `sum(rate(greptime_mito_gc_runs_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_files_deleted_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. | `prometheus` | `short` | `runs` | +| Mito GC Duration | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | P99 and average Mito garbage-collection duration by stage. | `prometheus` | `s` | `{{stage}}-p99` | +# Storage | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | | Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request Wait P99 and Avg per Worker | `histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{worker}}]-p95` | | Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | | Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | -| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | -| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | | Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | -| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | -| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | -| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | -| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| Read Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Read Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Write Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Write Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | | Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | -| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | -| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`
`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` | | Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`
`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` | -| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` | +| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Active series and field-builder counts per memtable by instance. | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` | | Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`
`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` | | Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -# OpenDAL +# Flush and Compaction | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` | -# Remote WAL +| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | +| Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` | +| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | +| Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`
`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` | +# Index | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | -| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | -| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` | -| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` | +| Index Apply Elapsed Time | `histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. | `prometheus` | `s` | `{{type}}-p95` | +| Index Create Elapsed Time | `histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. | `prometheus` | `s` | `{{type}}-{{stage}}-p95` | +| Index Create Rows and Bytes | `sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))`
`sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))` | `timeseries` | Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. | `prometheus` | `rowsps` | `{{type}}-rows` | +| Index Memory Usage | `greptime_index_apply_memory_usage`
`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` | +| Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` | +| Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` | +| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` | # Metasrv | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | +| Inactive and Lease-expired Regions | `sum(greptime_meta_inactive_regions)`
`sum(greptime_lease_expired_region{instance=~"$datanode"})` | `timeseries` | Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. | `prometheus` | `short` | `inactive-regions` | +| Heartbeat Health | `sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))`
`sum(greptime_meta_heartbeat_connection_num)`
`sum(rate(greptime_frontend_heartbeat_send_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_heartbeat_recv_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_send_count{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_recv_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. | `prometheus` | `short` | `meta-recv-rate` | | Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` | -| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` | +| Region migration error | `rate(greptime_meta_region_migration_error[$__rate_interval])` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` | | Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` | | Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` | -| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` | +| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))`
`sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` | | Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) ( rate(greptime_meta_handler_execute_bucket[$__rate_interval]) -))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` | -| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` | -| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` | -| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` | -| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` | -| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` | -| Reconciliation stats | `greptime_meta_reconciliation_stats` | `timeseries` | Reconciliation stats | `prometheus` | `s` | `{{pod}}-{{table_type}}-{{type}}` | -| Reconciliation steps | `histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` | -# Flownode -| Title | Query | Type | Description | Datasource | Unit | Legend Format | -| --- | --- | --- | --- | --- | --- | --- | -| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | -| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` | -| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | -| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | -| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` | -# Trigger -| Title | Query | Type | Description | Datasource | Unit | Legend Format | -| --- | --- | --- | --- | --- | --- | --- | -| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` | -| Trigger Eval Elapsed | `histogram_quantile(0.99, - rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]) -)`
`histogram_quantile(0.75, - rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]) -)` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | -| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` | -| Send Alert Elapsed | `histogram_quantile(0.99, - rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]) -)`
`histogram_quantile(0.75, - rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]) -)` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99` | -| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` | -| Save Alert Elapsed | `histogram_quantile(0.99, - rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]) -)`
`histogram_quantile(0.75, - rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]) -)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` | -| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` | +))`
`sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` | +| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` | +| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Rate of heartbeats received by metasrv from datanodes and frontends. | `prometheus` | `s` | `{{pod}}` | +| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))`
`sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average latency of metasrv key-value store operations by op and target. | `prometheus` | `s` | `{{pod}}-{{op}} p99` | +| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Rate of metasrv key-value store operations by op. | `prometheus` | `none` | `{{pod}}-{{op}} p99` | +| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))` | `timeseries` | p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` | +| Reconciliation stats | `rate(greptime_meta_reconciliation_stats[$__rate_interval])` | `timeseries` | Reconciliation stats | `prometheus` | `ops` | `{{pod}}-{{table_type}}-{{type}}` | +| Reconciliation steps | `histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` | # Hotspot | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | @@ -240,3 +250,45 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data | Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` | | Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` | | Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` | +# Object Store +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Write P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Other Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` | +# WAL +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | +| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | +| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | +| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` | +| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` | +# Flownode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | +| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` | +| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | +| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` | +# Trigger +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` | +| Trigger Eval Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `p99` | +| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` | +| Send Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `p99` | +| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` | +| Save Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `p99` | +| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` | diff --git a/grafana/dashboards/metrics/cluster/dashboard.yaml b/grafana/dashboards/metrics/cluster/dashboard.yaml index ca8f88267f4..127a8d606b8 100644 --- a/grafana/dashboards/metrics/cluster/dashboard.yaml +++ b/grafana/dashboards/metrics/cluster/dashboard.yaml @@ -29,24 +29,36 @@ groups: type: prometheus uid: ${metrics} legendFormat: __auto - - title: Total Storage Size + - title: Total Query Rate type: stat - description: Total number of data file size. - unit: decbytes + description: Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. + unit: reqps queries: - - expr: select SUM(disk_size) from information_schema.region_statistics; + - expr: sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) datasource: - type: mysql - uid: ${information_schema} - - title: Total Rows + type: prometheus + uid: ${metrics} + legendFormat: queries + - title: User-facing Error Rate type: stat - description: Total number of data rows in the cluster. Calculated by sum of rows from each region. - unit: sishort + description: Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. + unit: eps queries: - - expr: select SUM(region_rows) from information_schema.region_statistics; + - expr: sum(rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval])) datasource: - type: mysql - uid: ${information_schema} + type: prometheus + uid: ${metrics} + legendFormat: errors + - title: Recent Restarts + type: stat + description: Process restarts over the selected time range across GreptimeDB roles. + unit: short + queries: + - expr: sum(changes(process_start_time_seconds[$__range])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: restarts - title: Deployment type: stat description: The deployment topology of GreptimeDB. @@ -87,6 +99,24 @@ groups: datasource: type: mysql uid: ${information_schema} + - title: Total Storage Size + type: stat + description: Total number of data file size. + unit: decbytes + queries: + - expr: select SUM(disk_size) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Total Rows + type: stat + description: Total number of data rows in the cluster. Calculated by sum of rows from each region. + unit: sishort + queries: + - expr: select SUM(region_rows) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} - title: Data Size type: stat description: The data size of wal/index/manifest in the GreptimeDB. @@ -104,6 +134,56 @@ groups: datasource: type: mysql uid: ${information_schema} + - title: Total Ingestion Rate Trend + type: timeseries + description: Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: ingestion + - title: Total Query Rate Trend + type: timeseries + description: Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. + unit: reqps + queries: + - expr: sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: queries + - title: HTTP Request P99 and Avg + type: timeseries + description: Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-p99 + - expr: sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-avg + - title: gRPC Request P99 and Avg + type: timeseries + description: Tail and average latency for gRPC requests served by frontends. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-p99 + - expr: sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-avg - title: Ingestion panels: - title: Total Ingestion Rate @@ -123,71 +203,478 @@ groups: type: prometheus uid: ${metrics} legendFormat: ingestion - - title: Ingestion Rate by Type + - title: Ingestion Rate by Protocol type: timeseries - description: | - Total ingestion rate. - - Here we listed 3 primary protocols: - - - Prometheus remote write - - Greptime's gRPC API (when using our ingest SDK) - - Log ingestion http API + description: Rows, samples, or documents ingested by primary observability and table-ingestion protocols. unit: rowsps queries: - - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval])) + - expr: sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: http-logs - - expr: sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval])) + legendFormat: table-operator + - expr: sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~"$frontend"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: prometheus-remote-write - - title: Queries - panels: - - title: Total Query Rate + - expr: sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-logs + - expr: sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: loki-logs + - expr: sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: elasticsearch-docs + - expr: sum(rate(greptime_frontend_otlp_metrics_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-metrics + - expr: sum(rate(greptime_frontend_otlp_logs_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-logs + - expr: sum(rate(greptime_frontend_otlp_traces_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-traces + - title: Ingestion Latency by Protocol type: timeseries - description: |- - Total rate of query API calls by protocol. This metric is collected from frontends. - - Here we listed 3 main protocols: - - MySQL - - Postgres - - Prometheus API - - Note that there are some other minor query APIs like /sql are not included - unit: reqps + description: p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. + unit: s queries: - - expr: sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: mysql - - expr: sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + legendFormat: prometheus-write + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: pg - - expr: sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval])) + legendFormat: http-logs + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: promql - - title: Resources - panels: - - title: Datanode Memory per Instance + legendFormat: loki-logs + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: elasticsearch + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-metrics + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-logs + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-traces + - expr: sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: prom-write-avg + - expr: sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-logs-avg + - expr: sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: loki-logs-avg + - expr: sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: elasticsearch-avg + - expr: sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-metrics-avg + - expr: sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-logs-avg + - expr: sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-traces-avg + - title: Bulk Insert Message Rows and Size type: timeseries - description: Current memory usage by instance + description: Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: rows + - expr: sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: bytes + - title: Prom Store Flush Pipeline + type: timeseries + description: Remote-write pending-row flush operations, flushed rows, and p99 flush latency. + unit: short + queries: + - expr: sum(rate(greptime_prom_store_flush_total{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-ops + - expr: sum(rate(greptime_prom_store_flush_rows_sum{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-rows + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-p99 + - title: OTLP Trace Failures + type: timeseries + description: OTLP trace ingestion failures reported by frontends. + unit: eps + queries: + - expr: sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trace-failures + - title: Health + panels: + - title: Protocol Error Rates + type: timeseries + description: User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. + unit: eps + queries: + - expr: sum by (protocol) (rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: server-{{protocol}} + - expr: sum by (code) (rate(greptime_servers_auth_failure_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: auth-{{code}} + - expr: sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics",code!~"2.."}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-{{method}}-{{path}}-{{code}} + - expr: sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend",code!~"0|OK"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-{{path}}-{{code}} + - title: Frontend and Query Rejections + type: timeseries + description: Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. + unit: rps + queries: + - expr: sum(rate(greptime_servers_request_memory_rejected_total{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: request-memory + - expr: sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: query-memory + - title: Datanode Write Failures + type: timeseries + description: Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. + unit: eps + queries: + - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: region-request-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: failed-insert-[{{instance}}]-[{{pod}}] + - title: Buffered Ingestion Loss + type: timeseries + description: Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. + unit: eps + queries: + - expr: sum(rate(greptime_pending_rows_flush_failures{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-failures + - expr: sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: dropped-rows + - title: Mito Backpressure and Failures + type: timeseries + description: Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. + unit: eps + queries: + - expr: sum(rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: write-reject + - expr: sum(rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: write-stall + - expr: sum(rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-failure + - expr: sum(rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-failure + - title: Scan and Compaction Memory Rejects + type: timeseries + description: Datanode scan and compaction memory rejection/exhaustion counters. + unit: rps + queries: + - expr: sum(rate(greptime_mito_scan_requests_rejected_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: scan-rejected + - expr: sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: scan-exhausted + - expr: sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-rejected + - title: OpenDAL Errors + type: timeseries + description: Object-store errors by scheme, operation, and error, excluding NotFound noise. + unit: eps + queries: + - expr: sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode",error!="NotFound"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{scheme}}-{{operation}}-{{error}}' + - title: Metasrv Failures + type: timeseries + description: Region migration and reconciliation failures in metasrv. + unit: eps + queries: + - expr: sum(rate(greptime_meta_region_migration_fail[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: migration-fail + - expr: sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: reconciliation-error + - title: Flow and Trigger Failures + type: timeseries + description: Derived-data and alerting pipeline failures. + unit: eps + queries: + - expr: sum by (code) (rate(greptime_flow_errors[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flow-{{code}} + - expr: sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trigger-evaluate + - expr: sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trigger-send-alert + - expr: sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trigger-save-alert + - title: Mito GC Failures + type: timeseries + description: Mito garbage-collection errors and skipped/orphaned files on datanodes. + unit: short + queries: + - expr: sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: gc-errors + - expr: sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: orphaned-index-files + - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: skipped-unparsable + - title: Capacity + panels: + - title: Runtime Threads + type: timeseries + description: Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_runtime_threads_alive) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: alive-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_runtime_threads_idle) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: idle-[{{instance}}]-[{{pod}}] + - title: Request Memory Utilization + type: timeseries + description: Frontend request memory usage divided by configured request memory limit. + unit: percentunit + queries: + - expr: sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~"$frontend"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~"$frontend"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Query Memory Usage + type: timeseries + description: Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. unit: bytes queries: - - expr: sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod) + - expr: sum by (instance, pod) (greptime_query_memory_pool_usage_bytes) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{ pod }}]' - - expr: max(greptime_memory_limit_in_bytes{instance=~"$datanode"}) + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Scan and Compaction Memory + type: timeseries + description: Datanode scan memory usage and compaction memory utilization. + unit: bytes + queries: + - expr: sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: scan-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-used-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-limit-[{{instance}}]-[{{pod}}] + - title: Write Buffer and Active Stalling + type: timeseries + description: Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. + unit: bytes + queries: + - expr: sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: buffer-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: stalling-[{{instance}}]-[{{pod}}] + - title: Prom Store Backlog + type: timeseries + description: Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~"$frontend"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: rows-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~"$frontend"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: batches-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~"$frontend"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: workers-[{{instance}}]-[{{pod}}] + - title: Inflight Flush and Compaction + type: timeseries + description: Current in-flight flush and compaction tasks on datanodes. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-[{{instance}}]-[{{pod}}] + - title: Resources + panels: + - title: Frontend CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu' + - expr: max(greptime_cpu_limit_in_millicores{instance=~"$frontend"}) datasource: type: prometheus uid: ${metrics} @@ -207,6 +694,21 @@ groups: type: prometheus uid: ${metrics} legendFormat: limit + - title: Metasrv CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - expr: max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: limit - title: Frontend Memory per Instance type: timeseries description: Current memory usage by instance @@ -222,17 +724,17 @@ groups: type: prometheus uid: ${metrics} legendFormat: limit - - title: Frontend CPU Usage per Instance + - title: Datanode Memory per Instance type: timeseries - description: Current cpu usage by instance - unit: none + description: Current memory usage by instance + unit: bytes queries: - - expr: sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod) + - expr: sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu' - - expr: max(greptime_cpu_limit_in_millicores{instance=~"$frontend"}) + legendFormat: '[{{instance}}]-[{{ pod }}]' + - expr: max(greptime_memory_limit_in_bytes{instance=~"$datanode"}) datasource: type: prometheus uid: ${metrics} @@ -252,17 +754,17 @@ groups: type: prometheus uid: ${metrics} legendFormat: limit - - title: Metasrv CPU Usage per Instance + - title: Flownode CPU Usage per Instance type: timeseries description: Current cpu usage by instance unit: none queries: - - expr: sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod) + - expr: sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"}) + - expr: max(greptime_cpu_limit_in_millicores{instance=~"$flownode"}) datasource: type: prometheus uid: ${metrics} @@ -282,21 +784,168 @@ groups: type: prometheus uid: ${metrics} legendFormat: limit - - title: Flownode CPU Usage per Instance + - title: Queries + panels: + - title: Query Rate by Protocol type: timeseries - description: Current cpu usage by instance - unit: none + description: Query API call rates by protocol, collected from frontends. + unit: reqps queries: - - expr: sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod) + - expr: sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_cpu_limit_in_millicores{instance=~"$flownode"}) + legendFormat: mysql + - expr: sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: limit + legendFormat: postgres + - expr: sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql + - expr: sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-sql + - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-query + - title: Query Latency by Protocol + type: timeseries + description: p95, p99, and average query latency by main frontend protocol. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-p95 + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-p99 + - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-p95 + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-p99 + - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql-p95 + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql-p99 + - expr: sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-avg + - expr: sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-avg + - expr: sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql-avg + - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-query-avg + - title: Query Stage Latency + type: timeseries + description: p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p95-{{stage}} + - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99-{{stage}} + - title: Merge Scan Fan-out and Errors + type: timeseries + description: Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_query_merge_scan_regions) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: regions-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: errors-[{{instance}}]-[{{pod}}] + - title: Pushdown Fallback Errors + type: timeseries + description: Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. + unit: eps + queries: + - expr: sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: pushdown-fallback-errors + - title: PromQL Series Count + type: timeseries + description: Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_promql_series_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Connections and Prepared Statements + type: timeseries + description: MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~"$frontend"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-connections-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~"$frontend"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-connections-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-prepared-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-prepared-[{{instance}}]-[{{pod}}] - title: Frontend Requests panels: - title: HTTP QPS per Instance @@ -309,9 +958,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]' - - title: HTTP P99 per Instance + - title: HTTP P99 and Avg per Instance type: timeseries - description: HTTP P99 per Instance. + description: HTTP P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval]))) @@ -319,6 +968,11 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - expr: sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg' - title: gRPC QPS per Instance type: timeseries description: gRPC QPS per Instance. @@ -329,9 +983,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]' - - title: gRPC P99 per Instance + - title: gRPC P99 and Avg per Instance type: timeseries - description: gRPC P99 per Instance. + description: gRPC P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) @@ -339,6 +993,11 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg' - title: MySQL QPS per Instance type: timeseries description: MySQL QPS per Instance. @@ -349,9 +1008,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: MySQL P99 per Instance + - title: MySQL P99 and Avg per Instance type: timeseries - description: MySQL P99 per Instance. + description: MySQL P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) @@ -359,6 +1018,11 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]-p99' + - expr: sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-avg' - title: PostgreSQL QPS per Instance type: timeseries description: PostgreSQL QPS per Instance. @@ -369,9 +1033,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: PostgreSQL P99 per Instance + - title: PostgreSQL P99 and Avg per Instance type: timeseries - description: PostgreSQL P99 per Instance. + description: PostgreSQL P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval]))) @@ -379,18 +1043,13 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - title: Frontend to Datanode - panels: - - title: Ingest Rows per Instance - type: timeseries - description: Ingestion rate by row as in each frontend - unit: rowsps - queries: - - expr: sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval])) + - expr: sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' + legendFormat: '[{{instance}}]-[{{pod}}]-avg' + - title: Frontend to Datanode + panels: - title: Region Call QPS per Instance type: timeseries description: Region Call QPS per Instance. @@ -401,9 +1060,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' - - title: Region Call P99 per Instance + - title: Region Call P99 and Avg per Instance type: timeseries - description: Region Call P99 per Instance. + description: Region Call P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval]))) @@ -411,7 +1070,12 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' - - title: 'Frontend Handle Bulk Insert Elapsed Time ' + - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg' + - title: Frontend Handle Bulk Insert Elapsed Time type: timeseries description: Per-stage time for frontend to handle bulk insert requests unit: s @@ -426,7 +1090,104 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95' - - title: Mito Engine + - title: Datanode + panels: + - title: Region Request Failures and Failed Inserts + type: timeseries + description: Datanode region request failures and failed inserts by instance. + unit: eps + queries: + - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: request-fail-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: failed-insert-[{{instance}}]-[{{pod}}] + - title: Write Rejects and Stalls + type: timeseries + description: Mito write rejects, write stall events, and active write stalling by datanode. + unit: short + queries: + - expr: sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: reject-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: stall-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: active-stalling-[{{instance}}]-[{{pod}}] + - title: Flush and Compaction Failures + type: timeseries + description: Mito flush and compaction failure rates by datanode. + unit: eps + queries: + - expr: sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-[{{instance}}]-[{{pod}}] + - title: Mito GC Health + type: timeseries + description: Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. + unit: short + queries: + - expr: sum(rate(greptime_mito_gc_runs_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: runs + - expr: sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: errors + - expr: sum(rate(greptime_mito_gc_files_deleted_total{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: files-deleted + - expr: sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: orphaned-index-files + - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: skipped-unparsable + - title: Mito GC Duration + type: timeseries + description: P99 and average Mito garbage-collection duration by stage. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{stage}}-p99' + - expr: sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{stage}}-avg' + - title: Storage panels: - title: Request OPS per Instance type: timeseries @@ -438,9 +1199,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' - - title: Request P99 per Instance + - title: Request P99 and Avg per Instance type: timeseries - description: Request P99 per Instance. + description: Request P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) @@ -448,6 +1209,31 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg' + - title: Request Wait P99 and Avg per Worker + type: timeseries + description: Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p95' + - expr: histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p99' + - expr: sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-avg' - title: Write Buffer per Instance type: timeseries description: Write Buffer per Instance. @@ -468,25 +1254,6 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Flush OPS per Instance - type: timeseries - description: Flush QPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]' - - title: Write Stall per Instance - type: timeseries - description: Write Stall per Instance. - queries: - - expr: sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"}) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - title: Read Stage OPS per Instance type: timeseries description: Read Stage OPS per Instance. @@ -497,9 +1264,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Read Stage P99 per Instance + - title: Read Stage P99 and Avg per Instance type: timeseries - description: Read Stage P99 per Instance. + description: Read Stage P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) @@ -507,9 +1274,14 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' - - title: Write Stage P99 per Instance + - expr: sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg' + - title: Write Stage P99 and Avg per Instance type: timeseries - description: Write Stage P99 per Instance. + description: Write Stage P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) @@ -517,61 +1289,11 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' - - title: Compaction OPS per Instance - type: timeseries - description: Compaction OPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{pod}}]' - - title: Compaction Elapsed Time per Instance by Stage - type: timeseries - description: Compaction latency by stage - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99' - - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + - expr: sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg' - - title: Compaction P99 per Instance - type: timeseries - description: Compaction P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction' - - title: WAL write size - type: timeseries - description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. - unit: bytes - queries: - - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95' - - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99' - - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-throughput' - title: Cached Bytes per Instance type: timeseries description: Cached Bytes per Instance. @@ -582,61 +1304,6 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' - - title: Inflight Compaction - type: timeseries - description: Ongoing compaction task count - unit: none - queries: - - expr: greptime_mito_inflight_compaction_count - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: WAL sync duration seconds - type: timeseries - description: Raft engine (local disk) log store sync latency, p99 - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - title: Log Store op duration seconds - type: timeseries - description: Write-ahead log operations latency at p99 - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99' - - title: Inflight Flush - type: timeseries - description: Ongoing flush task count - unit: none - queries: - - expr: greptime_mito_inflight_flush_count - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Compaction Input/Output Bytes - type: timeseries - description: Compaction oinput output bytes - unit: bytes - queries: - - expr: sum by(instance, pod) (greptime_mito_compaction_input_bytes) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-input' - - expr: sum by(instance, pod) (greptime_mito_compaction_output_bytes) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-output' - title: Region Worker Handle Bulk Insert Requests type: timeseries description: Per-stage elapsed time for region worker to handle bulk insert region requests. @@ -654,7 +1321,7 @@ groups: legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG' - title: Active Series and Field Builders Count type: timeseries - description: Compaction oinput output bytes + description: Active series and field-builder counts per memtable by instance. unit: none queries: - expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count) @@ -691,161 +1358,282 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' - - title: OpenDAL + - title: Flush and Compaction panels: - - title: QPS per Instance + - title: Flush OPS per Instance type: timeseries - description: QPS per Instance. + description: Flush QPS per Instance. unit: ops queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval])) + - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Read QPS per Instance + legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]' + - title: Flush Elapsed Time type: timeseries - description: Read QPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Read P99 per Instance - type: timeseries - description: Read P99 per Instance. + description: Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read|Reader::read"}[$__rate_interval]))) + - expr: histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Write QPS per Instance + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' + - expr: histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - expr: sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg' + - title: Flush Throughput type: timeseries - description: Write QPS per Instance. - unit: ops + description: Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. + unit: Bps queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) + - expr: sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Write P99 per Instance - type: timeseries - description: Write P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write|Writer::close|write"}[$__rate_interval]))) + legendFormat: '[{{instance}}]-[{{pod}}]-bytes' + - expr: sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~"$datanode"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: List QPS per Instance + legendFormat: '[{{instance}}]-[{{pod}}]-files' + - title: Inflight Flush type: timeseries - description: List QPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' - - title: List P99 per Instance - type: timeseries - description: List P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' - - title: Other Requests per Instance - type: timeseries - description: Other Requests per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read|write|list|stat"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Other Request P99 per Instance - type: timeseries - description: Other Request P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|write|list|Writer::write|Writer::close|Reader::read"}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Opendal traffic - type: timeseries - description: Total traffic as in bytes by instance and operation - unit: decbytes - queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: OpenDAL errors per Instance - type: timeseries - description: OpenDAL error counts per Instance. - queries: - - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]' - - title: Remote WAL - panels: - - title: Triggered region flush total - type: timeseries - description: Triggered region flush total + description: Ongoing flush task count unit: none queries: - - expr: meta_triggered_region_flush_total + - expr: greptime_mito_inflight_flush_count datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{topic_name}}' - - title: Triggered region checkpoint total + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Compaction OPS per Instance type: timeseries - description: Triggered region checkpoint total + description: Compaction OPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{pod}}]' + - title: Inflight Compaction + type: timeseries + description: Ongoing compaction task count unit: none queries: - - expr: meta_triggered_region_checkpoint_total + - expr: greptime_mito_inflight_compaction_count datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{topic_name}}' - - title: Topic estimated replay size + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Compaction P99 and Avg per Instance type: timeseries - description: Topic estimated max replay size + description: Compaction P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-avg' + - title: Compaction Elapsed Time per Instance by Stage + type: timeseries + description: Compaction latency by stage + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99' + - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg' + - title: Compaction Input/Output Bytes + type: timeseries + description: Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. + unit: Bps + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-input' + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-output' + - title: Index + panels: + - title: Index Apply Elapsed Time + type: timeseries + description: Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-p95' + - expr: histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-p99' + - title: Index Create Elapsed Time + type: timeseries + description: Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-{{stage}}-p95' + - expr: histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-{{stage}}-p99' + - title: Index Create Rows and Bytes + type: timeseries + description: Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. + unit: rowsps + queries: + - expr: sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-rows' + - expr: sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-bytes' + - title: Index Memory Usage + type: timeseries + description: Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. unit: bytes queries: - - expr: meta_topic_estimated_replay_size + - expr: greptime_index_apply_memory_usage datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{topic_name}}' - - title: Kafka logstore's bytes traffic + legendFormat: apply + - expr: sum by (type) (greptime_index_create_memory_usage) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: create-{{type}} + - title: Index IO Bytes type: timeseries - description: Kafka logstore's bytes traffic - unit: bytes + description: Index read and write byte rates by operation and file type for puffin and intermediate files. + unit: Bps queries: - - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval]) + - expr: sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{logstore}}' + legendFormat: '{{type}}-{{file_type}}' + - title: Index IO Operations + type: timeseries + description: Index IO operation rates by operation and file type, including read, write, seek, and flush operations. + unit: ops + queries: + - expr: sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-{{file_type}}' + - title: Index Cache + type: timeseries + description: Index-related cache hits, misses, and evictions from Mito caches. + unit: ops + queries: + - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|vector_index|index_result"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: hit-{{type}} + - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|vector_index|index_result"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: miss-{{type}} + - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|vector_index|index_result"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: evict-{{type}}-{{cause}} - title: Metasrv panels: + - title: Inactive and Lease-expired Regions + type: timeseries + description: Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. + unit: short + queries: + - expr: sum(greptime_meta_inactive_regions) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: inactive-regions + - expr: sum(greptime_lease_expired_region{instance=~"$datanode"}) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: lease-expired + - title: Heartbeat Health + type: timeseries + description: Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. + unit: short + queries: + - expr: sum(rate(greptime_meta_heartbeat_rate[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: meta-recv-rate + - expr: sum(greptime_meta_heartbeat_connection_num) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: meta-connections + - expr: sum(rate(greptime_frontend_heartbeat_send_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: frontend-send + - expr: sum(rate(greptime_frontend_heartbeat_recv_count{instance=~"$frontend"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: frontend-recv + - expr: sum(rate(greptime_datanode_heartbeat_send_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-send + - expr: sum(rate(greptime_datanode_heartbeat_recv_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-recv - title: Region migration datanode type: status-history description: Counter of region migration by source and destination @@ -865,7 +1653,7 @@ groups: description: Counter of region migration error unit: none queries: - - expr: greptime_meta_region_migration_error + - expr: rate(greptime_meta_region_migration_error[$__rate_interval]) datasource: type: prometheus uid: ${metrics} @@ -900,6 +1688,11 @@ groups: type: prometheus uid: ${metrics} legendFormat: '{{pod}} {{op}} {{type}} {{result}} p90' + - expr: sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg' - title: Handler Execution Latency type: timeseries description: | @@ -914,20 +1707,25 @@ groups: type: prometheus uid: ${metrics} legendFormat: '{{pod}} {{name}} p90' + - expr: sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{name}}]-avg' - title: Heartbeat Packet Size type: timeseries description: | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload. unit: bytes queries: - - expr: histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket)) + - expr: histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '{{pod}}' - title: Meta Heartbeat Receive Rate type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: Rate of heartbeats received by metasrv from datanodes and frontends. unit: s queries: - expr: rate(greptime_meta_heartbeat_rate[$__rate_interval]) @@ -937,17 +1735,22 @@ groups: legendFormat: '{{pod}}' - title: Meta KV Ops Latency type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: p99 and average latency of metasrv key-value store operations by op and target. unit: s queries: - - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket)) + - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '{{pod}}-{{op}} p99' + - expr: sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{op}}]-[{{target}}]-avg' - title: Rate of meta KV Ops type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: Rate of metasrv key-value store operations by op. unit: none queries: - expr: rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]) @@ -957,35 +1760,35 @@ groups: legendFormat: '{{pod}}-{{op}} p99' - title: DDL Latency type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. unit: s queries: - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateLogicalTables-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateTable-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateView-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateFlow-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: DropTable-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} @@ -993,9 +1796,9 @@ groups: - title: Reconciliation stats type: timeseries description: Reconciliation stats - unit: s + unit: ops queries: - - expr: greptime_meta_reconciliation_stats + - expr: rate(greptime_meta_reconciliation_stats[$__rate_interval]) datasource: type: prometheus uid: ${metrics} @@ -1005,154 +1808,11 @@ groups: description: Elapsed of Reconciliation steps unit: s queries: - - expr: histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket) + - expr: histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '{{procedure_name}}-{{step}}-P90' - - title: Flownode - panels: - - title: Flow Ingest / Output Rate - type: timeseries - description: Flow Ingest / Output Rate. - queries: - - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]' - - title: Flow Ingest Latency - type: timeseries - description: Flow Ingest Latency. - queries: - - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p95' - - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - title: Flow Operation Latency - type: timeseries - description: Flow Operation Latency. - queries: - - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' - - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' - - title: Flow Buffer Size per Instance - type: timeseries - description: Flow Buffer Size per Instance. - queries: - - expr: greptime_flow_input_buf_size - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Flow Processing Error per Instance - type: timeseries - description: Flow Processing Error per Instance. - queries: - - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]' - - title: Trigger - panels: - - title: Trigger Count - type: timeseries - description: Total number of triggers currently defined. - queries: - - expr: greptime_trigger_count{} - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - - title: Trigger Eval Elapsed - type: timeseries - description: Elapsed time for trigger evaluation, including query execution and condition evaluation. - unit: s - queries: - - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p75' - - title: Trigger Eval Failure Rate - type: timeseries - description: Rate of failed trigger evaluations. - unit: none - queries: - - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval]) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - - title: Send Alert Elapsed - type: timeseries - description: Elapsed time to send trigger alerts to notification channels. - unit: s - queries: - - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99' - - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75' - - title: Send Alert Failure Rate - type: timeseries - description: Rate of failures when sending trigger alerts. - unit: none - queries: - - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval]) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - - title: Save Alert Elapsed - type: timeseries - description: Elapsed time to persist trigger alert records. - unit: s - queries: - - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99' - - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75' - - title: Save Alert Failure Rate - type: timeseries - description: Rate of failures when persisting trigger alert records. - unit: none - queries: - - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - title: Hotspot panels: - title: Hotspot Regions @@ -1310,3 +1970,379 @@ groups: type: prometheus uid: ${metrics} legendFormat: '{{stage}}' + - title: Object Store + panels: + - title: QPS per Instance + type: timeseries + description: QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Read QPS per Instance + type: timeseries + description: Read QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Read P99 and Avg per Instance + type: timeseries + description: Read P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read|Reader::read"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: Write QPS per Instance + type: timeseries + description: Write QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Write P99 and Avg per Instance + type: timeseries + description: Write P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write|Writer::close|write"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: List QPS per Instance + type: timeseries + description: List QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: List P99 and Avg per Instance + type: timeseries + description: List P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: Other Requests per Instance + type: timeseries + description: Other Requests per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Other Request P99 and Avg per Instance + type: timeseries + description: Other Request P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: Opendal traffic + type: timeseries + description: Total traffic as in bytes by instance and operation + unit: decbytes + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: OpenDAL errors per Instance + type: timeseries + description: OpenDAL error counts per Instance. + queries: + - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]' + - title: WAL + panels: + - title: WAL write size + type: timeseries + description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. + unit: bytes + queries: + - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95' + - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99' + - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-throughput' + - title: WAL sync duration seconds + type: timeseries + description: Raft engine (local disk) log store sync latency, p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Log Store op duration seconds + type: timeseries + description: Write-ahead log operations latency at p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99' + - title: Triggered region flush total + type: timeseries + description: Triggered region flush total + unit: none + queries: + - expr: meta_triggered_region_flush_total + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{topic_name}}' + - title: Triggered region checkpoint total + type: timeseries + description: Triggered region checkpoint total + unit: none + queries: + - expr: meta_triggered_region_checkpoint_total + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{topic_name}}' + - title: Topic estimated replay size + type: timeseries + description: Topic estimated max replay size + unit: bytes + queries: + - expr: meta_topic_estimated_replay_size + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{topic_name}}' + - title: Kafka logstore's bytes traffic + type: timeseries + description: Kafka logstore's bytes traffic + unit: bytes + queries: + - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{logstore}}' + - title: Flownode + panels: + - title: Flow Ingest / Output Rate + type: timeseries + description: Flow Ingest / Output Rate. + queries: + - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]' + - title: Flow Ingest Latency + type: timeseries + description: Flow Ingest Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - expr: sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-avg' + - title: Flow Operation Latency + type: timeseries + description: Flow Operation Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - title: Flow Buffer Size per Instance + type: timeseries + description: Flow Buffer Size per Instance. + queries: + - expr: greptime_flow_input_buf_size + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Flow Processing Error per Instance + type: timeseries + description: Flow Processing Error per Instance. + queries: + - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]' + - title: Trigger + panels: + - title: Trigger Count + type: timeseries + description: Total number of triggers currently defined. + queries: + - expr: greptime_trigger_count{} + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Trigger Eval Elapsed + type: timeseries + description: Elapsed time for trigger evaluation, including query execution and condition evaluation. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99 + - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p75 + - expr: sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: avg + - title: Trigger Eval Failure Rate + type: timeseries + description: Rate of failed trigger evaluations. + unit: none + queries: + - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Send Alert Elapsed + type: timeseries + description: Elapsed time to send trigger alerts to notification channels. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99 + - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p75 + - expr: sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: avg + - title: Send Alert Failure Rate + type: timeseries + description: Rate of failures when sending trigger alerts. + unit: none + queries: + - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Save Alert Elapsed + type: timeseries + description: Elapsed time to persist trigger alert records. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99 + - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p75 + - expr: sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: avg + - title: Save Alert Failure Rate + type: timeseries + description: Rate of failures when persisting trigger alert records. + unit: none + queries: + - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto diff --git a/grafana/dashboards/metrics/standalone/dashboard.json b/grafana/dashboards/metrics/standalone/dashboard.json index c8313ecb7a7..376e1665cd6 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.json +++ b/grafana/dashboards/metrics/standalone/dashboard.json @@ -36,7 +36,7 @@ "x": 0, "y": 0 }, - "id": 279, + "id": 393, "panels": [], "title": "Overview", "type": "row" @@ -68,12 +68,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, "x": 0, - "y": 1 + "y": 1, + "w": 3, + "h": 4 }, - "id": 265, + "id": 380, "options": { "colorMode": "value", "graphMode": "none", @@ -142,12 +142,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 2, "x": 3, - "y": 1 + "y": 1, + "w": 3, + "h": 4 }, - "id": 239, + "id": 381, "options": { "colorMode": "value", "graphMode": "area", @@ -227,12 +227,12 @@ "overrides": [] }, "gridPos": { - "h": 4, + "x": 6, + "y": 1, "w": 4, - "x": 5, - "y": 1 + "h": 4 }, - "id": 249, + "id": 382, "options": { "colorMode": "value", "graphMode": "none", @@ -270,16 +270,18 @@ }, { "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, - "description": "Total number of data file size.", + "description": "Total query API call rate across MySQL, PostgreSQL, and PromQL frontends.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "fieldMinMax": false, "mappings": [], + "max": 2, "thresholds": { "mode": "absolute", "steps": [ @@ -288,20 +290,20 @@ } ] }, - "unit": "decbytes" + "unit": "reqps" }, "overrides": [] }, "gridPos": { - "h": 4, + "x": 10, + "y": 1, "w": 4, - "x": 9, - "y": 1 + "h": 4 }, - "id": 248, + "id": 383, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -310,7 +312,8 @@ "lastNotNull" ], "fields": "", - "values": false + "limit": 1, + "values": true }, "showPercentChange": false, "textMode": "auto", @@ -319,72 +322,62 @@ "pluginVersion": "11.6.0", "targets": [ { - "dataset": "information_schema", "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "queries", + "range": true, + "refId": "A" } ], - "title": "Total Storage Size", + "title": "Total Query Rate", "type": "stat" }, { "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, - "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", + "description": "Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, + "fieldMinMax": false, "mappings": [], + "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" + }, + { + "color": "red", + "value": 0 } ] }, - "unit": "sishort" + "unit": "eps" }, "overrides": [] }, "gridPos": { - "h": 4, + "x": 14, + "y": 1, "w": 4, - "x": 13, - "y": 1 + "h": 4 }, - "id": 254, + "id": 384, "options": { "colorMode": "value", - "graphMode": "area", + "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", @@ -393,7 +386,8 @@ "lastNotNull" ], "fields": "", - "values": false + "limit": 1, + "values": true }, "showPercentChange": false, "textMode": "auto", @@ -402,36 +396,94 @@ "pluginVersion": "11.6.0", "targets": [ { - "dataset": "information_schema", "datasource": { - "type": "mysql", - "uid": "${information_schema}" + "type": "prometheus", + "uid": "${metrics}" }, "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "expr": "sum(rate(greptime_servers_error[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "errors", + "range": true, + "refId": "A" } ], - "title": "Total Rows", + "title": "User-facing Error Rate", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Process restarts over the selected time range across GreptimeDB roles.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 18, + "y": 1, + "w": 6, + "h": 4 + }, + "id": 385, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "limit": 1, + "values": true + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(changes(process_start_time_seconds[$__range]))", + "hide": false, + "instant": false, + "legendFormat": "restarts", + "range": true, + "refId": "A" + } + ], + "title": "Recent Restarts", "type": "stat" }, { @@ -462,12 +514,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 5, "x": 0, - "y": 5 + "y": 5, + "w": 6, + "h": 4 }, - "id": 243, + "id": 386, "options": { "colorMode": "value", "graphMode": "area", @@ -634,12 +686,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 5, - "x": 5, - "y": 5 + "x": 6, + "y": 5, + "w": 6, + "h": 4 }, - "id": 247, + "id": 387, "options": { "colorMode": "value", "graphMode": "area", @@ -782,6 +834,172 @@ "title": "Database Resources", "type": "stat" }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data file size.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 4, + "h": 4 + }, + "id": 388, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Storage Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "sishort" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 5, + "w": 4, + "h": 4 + }, + "id": 389, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Total Rows", + "type": "stat" + }, { "datasource": { "type": "mysql", @@ -807,12 +1025,12 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 5, - "x": 10, - "y": 5 + "x": 20, + "y": 5, + "w": 4, + "h": 4 }, - "id": 278, + "id": 390, "options": { "colorMode": "value", "graphMode": "area", @@ -926,15 +1144,449 @@ "title": "Data Size", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 9, + "w": 12, + "h": 6 + }, + "id": 538, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "ingestion", + "range": true, + "refId": "A" + } + ], + "title": "Total Ingestion Rate Trend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 9, + "w": 12, + "h": 6 + }, + "id": 539, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "queries", + "range": true, + "refId": "A" + } + ], + "title": "Total Query Rate Trend", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 15, + "w": 12, + "h": 6 + }, + "id": 392, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~\"/health|/metrics\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "http-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_requests_elapsed_sum{path!~\"/health|/metrics\"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-avg", + "range": true, + "refId": "B" + } + ], + "title": "HTTP Request P99 and Avg", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Tail and average latency for gRPC requests served by frontends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 15, + "w": 12, + "h": 6 + }, + "id": 557, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "grpc-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-avg", + "range": true, + "refId": "B" + } + ], + "title": "gRPC Request P99 and Avg", + "type": "timeseries" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 9 + "y": 21 }, - "id": 275, + "id": 419, "panels": [], "title": "Ingestion", "type": "row" @@ -1001,12 +1653,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 24, "x": 0, - "y": 10 + "y": 22, + "w": 8, + "h": 6 }, - "id": 193, + "id": 413, "options": { "legend": { "calcs": [ @@ -1030,7 +1682,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))", + "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "ingestion", @@ -1046,7 +1698,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", + "description": "Rows, samples, or documents ingested by primary observability and table-ingestion protocols.", "fieldConfig": { "defaults": { "color": { @@ -1103,12 +1755,12 @@ "overrides": [] }, "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 16 + "x": 8, + "y": 22, + "w": 8, + "h": 6 }, - "id": 277, + "id": 414, "options": { "legend": { "calcs": [ @@ -1132,12 +1784,12 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", "hide": false, "instant": false, - "legendFormat": "http-logs", + "legendFormat": "table-operator", "range": true, - "refId": "http_logs" + "refId": "A" }, { "datasource": { @@ -1150,10 +1802,4690 @@ "instant": false, "legendFormat": "prometheus-remote-write", "range": true, - "refId": "prometheus-remote-write" + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "loki-logs", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "elasticsearch-docs", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-metrics", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-logs", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-traces", + "range": true, + "refId": "H" } ], - "title": "Ingestion Rate by Type", + "title": "Ingestion Rate by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 22, + "w": 8, + "h": 6 + }, + "id": 415, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "prometheus-write", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "http-logs", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "loki-logs", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "elasticsearch", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "otlp-metrics", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "otlp-logs", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "otlp-traces", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "prom-write-avg", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-logs-avg", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "loki-logs-avg", + "range": true, + "refId": "J" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "elasticsearch-avg", + "range": true, + "refId": "K" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-metrics-avg", + "range": true, + "refId": "L" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-logs-avg", + "range": true, + "refId": "M" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "otlp-traces-avg", + "range": true, + "refId": "N" + } + ], + "title": "Ingestion Latency by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bytes" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 8, + "h": 6 + }, + "id": 416, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "rows", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "bytes", + "range": true, + "refId": "B" + } + ], + "title": "Bulk Insert Message Rows and Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Remote-write pending-row flush operations, flushed rows, and p99 flush latency.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 28, + "w": 8, + "h": 6 + }, + "id": 417, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_prom_store_flush_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-ops", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-rows", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "flush-p99", + "range": true, + "refId": "C" + } + ], + "title": "Prom Store Flush Pipeline", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "OTLP trace ingestion failures reported by frontends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 28, + "w": 8, + "h": 6 + }, + "id": 418, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trace-failures", + "range": true, + "refId": "A" + } + ], + "title": "OTLP Trace Failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 403, + "panels": [], + "title": "Health", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 35, + "w": 8, + "h": 6 + }, + "id": 394, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (protocol) (rate(greptime_servers_error[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "server-{{protocol}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "auth-{{code}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\",code!~\"2..\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-{{method}}-{{path}}-{{code}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~\"0|OK\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-{{path}}-{{code}}", + "range": true, + "refId": "D" + } + ], + "title": "Protocol Error Rates", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 35, + "w": 8, + "h": 6 + }, + "id": 395, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "request-memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "query-memory", + "range": true, + "refId": "B" + } + ], + "title": "Frontend and Query Rejections", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 35, + "w": 8, + "h": 6 + }, + "id": 396, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "region-request-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Datanode Write Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 8, + "h": 6 + }, + "id": 397, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_pending_rows_flush_failures[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-failures", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "dropped-rows", + "range": true, + "refId": "B" + } + ], + "title": "Buffered Ingestion Loss", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 41, + "w": 8, + "h": 6 + }, + "id": 398, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_write_reject_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "write-reject", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_write_stall_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "write-stall", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_flush_failure_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-failure", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_compaction_failure_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "compaction-failure", + "range": true, + "refId": "D" + } + ], + "title": "Mito Backpressure and Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Datanode scan and compaction memory rejection/exhaustion counters.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "rps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 41, + "w": 8, + "h": 6 + }, + "id": 399, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "scan-rejected", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "scan-exhausted", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "compaction-rejected", + "range": true, + "refId": "C" + } + ], + "title": "Scan and Compaction Memory Rejects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Object-store errors by scheme, operation, and error, excluding NotFound noise.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 47, + "w": 8, + "h": 6 + }, + "id": 400, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!=\"NotFound\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{scheme}}-{{operation}}-{{error}}", + "range": true, + "refId": "A" + } + ], + "title": "OpenDAL Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region migration and reconciliation failures in metasrv.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 47, + "w": 8, + "h": 6 + }, + "id": 401, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "migration-fail", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "reconciliation-error", + "range": true, + "refId": "B" + } + ], + "title": "Metasrv Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Derived-data and alerting pipeline failures.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 47, + "w": 8, + "h": 6 + }, + "id": 402, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (code) (rate(greptime_flow_errors[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flow-{{code}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trigger-evaluate", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trigger-send-alert", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "trigger-save-alert", + "range": true, + "refId": "D" + } + ], + "title": "Flow and Trigger Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito garbage-collection errors and skipped/orphaned files on datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 53, + "w": 8, + "h": 6 + }, + "id": 550, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "gc-errors", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "orphaned-index-files", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "skipped-unparsable", + "range": true, + "refId": "C" + } + ], + "title": "Mito GC Failures", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 59 + }, + "id": 411, + "panels": [], + "title": "Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 60, + "w": 8, + "h": 6 + }, + "id": 404, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_runtime_threads_alive)", + "hide": false, + "instant": false, + "legendFormat": "alive-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_runtime_threads_idle)", + "hide": false, + "instant": false, + "legendFormat": "idle-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Runtime Threads", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Frontend request memory usage divided by configured request memory limit.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 60, + "w": 8, + "h": 6 + }, + "id": 405, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Request Memory Utilization", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 60, + "w": 8, + "h": 6 + }, + "id": 406, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Query Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Datanode scan memory usage and compaction memory utilization.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 66, + "w": 8, + "h": 6 + }, + "id": 407, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes)", + "hide": false, + "instant": false, + "legendFormat": "scan-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes)", + "hide": false, + "instant": false, + "legendFormat": "compaction-used-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes)", + "hide": false, + "instant": false, + "legendFormat": "compaction-limit-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + } + ], + "title": "Scan and Compaction Memory", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 66, + "w": 8, + "h": 6 + }, + "id": 408, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_write_buffer_bytes)", + "hide": false, + "instant": false, + "legendFormat": "buffer-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count)", + "hide": false, + "instant": false, + "legendFormat": "stalling-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Write Buffer and Active Stalling", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 66, + "w": 8, + "h": 6 + }, + "id": 409, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_prom_store_pending_rows)", + "hide": false, + "instant": false, + "legendFormat": "rows-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_prom_store_pending_batches)", + "hide": false, + "instant": false, + "legendFormat": "batches-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_prom_store_pending_workers)", + "hide": false, + "instant": false, + "legendFormat": "workers-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + } + ], + "title": "Prom Store Backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current in-flight flush and compaction tasks on datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 72, + "w": 8, + "h": 6 + }, + "id": 410, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_inflight_flush_count)", + "hide": false, + "instant": false, + "legendFormat": "flush-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_inflight_compaction_count)", + "hide": false, + "instant": false, + "legendFormat": "compaction-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Inflight Flush and Compaction", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 412, + "title": "Resources", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 465, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Frontend CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 463, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Datanode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 467, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Metasrv CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 87 + }, + "id": 464, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Frontend Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 87 + }, + "id": 462, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes) by (instance, pod)", + "instant": false, + "legendFormat": "[{{instance}}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Datanode Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 87 + }, + "id": 466, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Metasrv Memory per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current cpu usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 95 + }, + "id": 469, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(greptime_cpu_limit_in_millicores)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Flownode CPU Usage per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Current memory usage by instance", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 95 + }, + "id": 468, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Mean", + "sortDesc": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes) by (instance, pod)", + "instant": false, + "legendFormat": "[{{ instance }}]-[{{ pod }}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "max(greptime_memory_limit_in_bytes)", + "hide": false, + "instant": false, + "legendFormat": "limit", + "range": true, + "refId": "B" + } + ], + "title": "Flownode Memory per Instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 427, + "panels": [], + "title": "Queries", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Query API call rates by protocol, collected from frontends.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 104, + "w": 8, + "h": 6 + }, + "id": 420, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "mysql", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "postgres", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http-sql", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-query", + "range": true, + "refId": "E" + } + ], + "title": "Query Rate by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "p95, p99, and average query latency by main frontend protocol.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 104, + "w": 8, + "h": 6 + }, + "id": 421, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "mysql-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "mysql-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "postgres-p95", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "postgres-p99", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "promql-p95", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "promql-p99", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "mysql-avg", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "postgres-avg", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql-avg", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "grpc-query-avg", + "range": true, + "refId": "J" + } + ], + "title": "Query Latency by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 104, + "w": 8, + "h": 6 + }, + "id": 422, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p95-{{stage}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99-{{stage}}", + "range": true, + "refId": "B" + } + ], + "title": "Query Stage Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 110, + "w": 8, + "h": 6 + }, + "id": 423, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_query_merge_scan_regions)", + "hide": false, + "instant": false, + "legendFormat": "regions-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "errors-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Merge Scan Fan-out and Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 110, + "w": 8, + "h": 6 + }, + "id": 424, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "pushdown-fallback-errors", + "range": true, + "refId": "A" + } + ], + "title": "Pushdown Fallback Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 110, + "w": 8, + "h": 6 + }, + "id": 425, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_promql_series_count)", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "PromQL Series Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 116, + "w": 8, + "h": 6 + }, + "id": 426, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_servers_mysql_connection_count)", + "hide": false, + "instant": false, + "legendFormat": "mysql-connections-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_servers_postgres_connection_count)", + "hide": false, + "instant": false, + "legendFormat": "postgres-connections-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "mysql-prepared-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "postgres-prepared-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "D" + } + ], + "title": "Connections and Prepared Statements", "type": "timeseries" }, { @@ -1162,1092 +6494,9 @@ "h": 1, "w": 24, "x": 0, - "y": 22 + "y": 122 }, - "id": 276, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 246 - }, - "id": 255, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))", - "instant": false, - "legendFormat": "mysql", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "pg", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "promql", - "range": true, - "refId": "C" - } - ], - "title": "Total Query Rate", - "type": "timeseries" - } - ], - "title": "Queries", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 23 - }, - "id": 274, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 247 - }, - "id": 256, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Datanode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 247 - }, - "id": 262, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Datanode CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 257 - }, - "id": 266, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Frontend Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 257 - }, - "id": 268, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Last *", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Frontend CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 267 - }, - "id": 269, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Metasrv Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 267 - }, - "id": 271, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Metasrv CPU Usage per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current memory usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 0, - "y": 277 - }, - "id": 272, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "max(greptime_memory_limit_in_bytes{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Flownode Memory per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Current cpu usage by instance", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 12, - "x": 12, - "y": 277 - }, - "id": 273, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Mean", - "sortDesc": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{ pod }}]", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "exemplar": false, - "expr": "max(greptime_cpu_limit_in_millicores{})", - "hide": false, - "instant": false, - "legendFormat": "limit", - "range": true, - "refId": "B" - } - ], - "title": "Flownode CPU Usage per Instance", - "type": "timeseries" - } - ], - "title": "Resources", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 24 - }, - "id": 280, + "id": 428, "panels": [ { "datasource": { @@ -2312,11 +6561,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 288 + "y": 123 }, - "id": 281, + "id": 470, "options": { "legend": { "calcs": [ @@ -2355,7 +6604,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "HTTP P99 per Instance.", + "description": "HTTP P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -2413,11 +6662,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 288 + "w": 8, + "x": 8, + "y": 123 }, - "id": 282, + "id": 471, "options": { "legend": { "calcs": [ @@ -2446,9 +6695,22 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~\"/health|/metrics\"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg", + "range": true, + "refId": "B" } ], - "title": "HTTP P99 per Instance", + "title": "HTTP P99 and Avg per Instance", "type": "timeseries" }, { @@ -2514,11 +6776,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 296 + "w": 8, + "x": 16, + "y": 123 }, - "id": 283, + "id": 472, "options": { "legend": { "calcs": [ @@ -2542,7 +6804,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))", + "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]", "range": true, @@ -2557,7 +6819,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "gRPC P99 per Instance.", + "description": "gRPC P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -2615,11 +6877,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 296 + "w": 8, + "x": 0, + "y": 131 }, - "id": 284, + "id": 473, "options": { "legend": { "calcs": [], @@ -2641,14 +6903,27 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg", + "range": true, + "refId": "B" } ], - "title": "gRPC P99 per Instance", + "title": "gRPC P99 and Avg per Instance", "type": "timeseries" }, { @@ -2714,11 +6989,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 304 + "w": 8, + "x": 8, + "y": 131 }, - "id": 285, + "id": 474, "options": { "legend": { "calcs": [ @@ -2742,7 +7017,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))", + "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, @@ -2757,7 +7032,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "MySQL P99 per Instance.", + "description": "MySQL P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -2815,11 +7090,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 304 + "w": 8, + "x": 16, + "y": 131 }, - "id": 286, + "id": 475, "options": { "legend": { "calcs": [ @@ -2844,14 +7119,27 @@ }, "editorMode": "code", "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "B" } ], - "title": "MySQL P99 per Instance", + "title": "MySQL P99 and Avg per Instance", "type": "timeseries" }, { @@ -2917,11 +7205,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 312 + "y": 139 }, - "id": 287, + "id": 476, "options": { "legend": { "calcs": [ @@ -2945,7 +7233,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))", + "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, @@ -2960,7 +7248,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "PostgreSQL P99 per Instance.", + "description": "PostgreSQL P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -3018,11 +7306,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 312 + "w": 8, + "x": 8, + "y": 139 }, - "id": 288, + "id": 477, "options": { "legend": { "calcs": [], @@ -3044,14 +7332,27 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "B" } ], - "title": "PostgreSQL P99 per Instance", + "title": "PostgreSQL P99 and Avg per Instance", "type": "timeseries" } ], @@ -3064,111 +7365,10 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 123 }, - "id": 289, + "id": 429, "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ingestion rate by row as in each frontend", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "rowsps" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 24, - "x": 0, - "y": 289 - }, - "id": 292, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Ingest Rows per Instance", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -3232,11 +7432,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 295 + "y": 124 }, - "id": 290, + "id": 479, "options": { "legend": { "calcs": [ @@ -3260,7 +7460,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))", + "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", "range": true, @@ -3275,7 +7475,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Region Call P99 per Instance.", + "description": "Region Call P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -3333,11 +7533,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 295 + "w": 8, + "x": 8, + "y": 124 }, - "id": 291, + "id": 480, "options": { "legend": { "calcs": [ @@ -3361,14 +7561,27 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg", + "range": true, + "refId": "B" } ], - "title": "Region Call P99 per Instance", + "title": "Region Call P99 and Avg per Instance", "type": "timeseries" }, { @@ -3430,43 +7643,15 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[minipc-1:4000]-[]-[encode]-AVG", - "[minipc-1:4000]-[]-[partition]-AVG", - "[minipc-1:4000]-[]-[decode_request]-AVG", - "[minipc-1:4000]-[]-[datanode_handle]-AVG" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 303 + "w": 8, + "x": 16, + "y": 124 }, - "id": 336, + "id": 481, "options": { "legend": { "calcs": [ @@ -3510,22 +7695,662 @@ "refId": "B" } ], - "title": "Frontend Handle Bulk Insert Elapsed Time ", + "title": "Frontend Handle Bulk Insert Elapsed Time", "type": "timeseries" } ], "title": "Frontend to Datanode", "type": "row" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 124 + }, + "id": 434, + "panels": [], + "title": "Datanode", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Datanode region request failures and failed inserts by instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 125, + "w": 8, + "h": 6 + }, + "id": 430, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "request-fail-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Region Request Failures and Failed Inserts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito write rejects, write stall events, and active write stalling by datanode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 125, + "w": 8, + "h": 6 + }, + "id": 431, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "reject-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "stall-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count)", + "hide": false, + "instant": false, + "legendFormat": "active-stalling-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "C" + } + ], + "title": "Write Rejects and Stalls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito flush and compaction failure rates by datanode.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 125, + "w": 8, + "h": 6 + }, + "id": 432, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "flush-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "compaction-[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "B" + } + ], + "title": "Flush and Compaction Failures", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 131, + "w": 8, + "h": 6 + }, + "id": 433, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_runs_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "runs", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "errors", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "files-deleted", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "orphaned-index-files", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "skipped-unparsable", + "range": true, + "refId": "E" + } + ], + "title": "Mito GC Health", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "P99 and average Mito garbage-collection duration by stage.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 131, + "w": 8, + "h": 6 + }, + "id": 551, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}-avg", + "range": true, + "refId": "B" + } + ], + "title": "Mito GC Duration", + "type": "timeseries" + }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 26 + "y": 137 }, - "id": 293, + "id": 435, "panels": [ { "datasource": { @@ -3590,11 +8415,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 193 + "y": 138 }, - "id": 294, + "id": 482, "options": { "legend": { "calcs": [ @@ -3618,7 +8443,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, @@ -3633,7 +8458,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Request P99 per Instance.", + "description": "Request P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -3691,11 +8516,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 193 + "w": 8, + "x": 8, + "y": 138 }, - "id": 295, + "id": 483, "options": { "legend": { "calcs": [ @@ -3719,14 +8544,155 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg", + "range": true, + "refId": "B" } ], - "title": "Request P99 per Instance", + "title": "Request P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 138 + }, + "id": 556, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-avg", + "range": true, + "refId": "C" + } + ], + "title": "Request Wait P99 and Avg per Worker", "type": "timeseries" }, { @@ -3792,11 +8758,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 225 + "y": 146 }, - "id": 296, + "id": 484, "options": { "legend": { "calcs": [ @@ -3820,7 +8786,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "greptime_mito_write_buffer_bytes{}", + "expr": "greptime_mito_write_buffer_bytes", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, @@ -3893,11 +8859,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 225 + "w": 8, + "x": 8, + "y": 146 }, - "id": 297, + "id": 485, "options": { "legend": { "calcs": [ @@ -3921,7 +8887,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))", + "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, @@ -3931,207 +8897,6 @@ "title": "Write Rows per Instance", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Flush QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 233 - }, - "id": 298, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", - "range": true, - "refId": "A" - } - ], - "title": "Flush OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stall per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 233 - }, - "id": 299, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write Stall per Instance", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -4195,11 +8960,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 241 + "w": 8, + "x": 16, + "y": 146 }, - "id": 300, + "id": 488, "options": { "legend": { "calcs": [ @@ -4223,7 +8988,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))", + "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage=\"total\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, @@ -4238,133 +9003,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Read Stage P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 241 - }, - "id": 301, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write Stage P99 per Instance.", + "description": "Read Stage P99 and average per Instance.", "fieldConfig": { "defaults": { "color": { @@ -4422,11 +9061,125 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 249 + "y": 154 }, - "id": 302, + "id": 489, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Read Stage P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write Stage P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 154 + }, + "id": 490, "options": { "legend": { "calcs": [ @@ -4452,213 +9205,11 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", "range": true, "refId": "A" - } - ], - "title": "Write Stage P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction OPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 249 - }, - "id": 303, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{ instance }}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Compaction OPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction latency by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 257 - }, - "id": 304, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", - "range": true, - "refId": "A" }, { "datasource": { @@ -4666,7 +9217,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", @@ -4674,235 +9225,7 @@ "refId": "B" } ], - "title": "Compaction Elapsed Time per Instance by Stage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 257 - }, - "id": 305, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", - "range": true, - "refId": "A" - } - ], - "title": "Compaction P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 265 - }, - "id": 306, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", - "range": true, - "refId": "C" - } - ], - "title": "WAL write size", + "title": "Write Stage P99 and Avg per Instance", "type": "timeseries" }, { @@ -4964,40 +9287,15 @@ }, "unit": "decbytes" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 265 + "w": 8, + "x": 16, + "y": 154 }, - "id": 307, + "id": 495, "options": { "legend": { "calcs": [], @@ -5019,7 +9317,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "greptime_mito_cache_bytes{}", + "expr": "greptime_mito_cache_bytes", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, @@ -5029,524 +9327,6 @@ "title": "Cached Bytes per Instance", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing compaction task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 273 - }, - "id": 308, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_compaction_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Compaction", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Raft engine (local disk) log store sync latency, p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 273 - }, - "id": 310, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "WAL sync duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write-ahead log operations latency at p99", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 281 - }, - "id": 311, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", - "range": true, - "refId": "A" - } - ], - "title": "Log Store op duration seconds", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Ongoing flush task count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 281 - }, - "id": 312, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_mito_inflight_flush_count", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]", - "range": true, - "refId": "A" - } - ], - "title": "Inflight Flush", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Compaction oinput output bytes", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 289 - }, - "id": 335, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-input", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-output", - "range": true, - "refId": "B" - } - ], - "title": "Compaction Input/Output Bytes", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -5606,44 +9386,15 @@ }, "unit": "s" }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[bulk_extend]-AVG", - "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", - "[minipc-1:4000]-[]-[process_bulk_req]-AVG", - "[minipc-1:4000]-[]-[write_bulk]-AVG" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 289 + "w": 8, + "x": 0, + "y": 162 }, - "id": 337, + "id": 501, "options": { "legend": { "calcs": [ @@ -5695,7 +9446,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Compaction oinput output bytes", + "description": "Active series and field-builder counts per memtable by instance.", "fieldConfig": { "defaults": { "color": { @@ -5753,11 +9504,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 297 + "w": 8, + "x": 8, + "y": 162 }, - "id": 348, + "id": 502, "options": { "legend": { "calcs": [ @@ -5867,11 +9618,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 297 + "w": 8, + "x": 16, + "y": 162 }, - "id": 338, + "id": 503, "options": { "legend": { "calcs": [ @@ -5984,11 +9735,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 305 + "y": 170 }, - "id": 349, + "id": 504, "options": { "legend": { "calcs": [ @@ -6012,7 +9763,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))", + "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval]))", "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" @@ -6022,7 +9773,7 @@ "type": "timeseries" } ], - "title": "Mito Engine", + "title": "Storage", "type": "row" }, { @@ -6031,16 +9782,16 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 138 }, - "id": 313, + "id": 555, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "QPS per Instance.", + "description": "Flush QPS per Instance.", "fieldConfig": { "defaults": { "color": { @@ -6053,6 +9804,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6096,12 +9848,12 @@ "overrides": [] }, "gridPos": { - "h": 10, - "w": 24, + "h": 8, + "w": 8, "x": 0, - "y": 459 + "y": 139 }, - "id": 314, + "id": 486, "options": { "legend": { "calcs": [ @@ -6117,7 +9869,7 @@ "sort": "desc" } }, - "pluginVersion": "12.0.0", + "pluginVersion": "11.6.0", "targets": [ { "datasource": { @@ -6125,14 +9877,14 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))", + "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval]))", "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", "range": true, "refId": "A" } ], - "title": "QPS per Instance", + "title": "Flush OPS per Instance", "type": "timeseries" }, { @@ -6140,7 +9892,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Read QPS per Instance.", + "description": "Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs.", "fieldConfig": { "defaults": { "color": { @@ -6153,6 +9905,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -6191,823 +9944,21 @@ } ] }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 469 - }, - "id": 315, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~\"read|Reader::read\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Read P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 469 + "h": 8, + "w": 8, + "x": 8, + "y": 139 }, - "id": 316, + "id": 540, "options": { "legend": { "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~\"read|Reader::read\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Read P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 476 - }, - "id": 317, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 476 - }, - "id": 318, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Write P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List QPS per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 483 - }, - "id": 319, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"list\"}[$__rate_interval]))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List QPS per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "List P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 483 - }, - "id": 320, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"list\"}[$__rate_interval])))", - "instant": false, - "interval": "", - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", - "range": true, - "refId": "A" - } - ], - "title": "List P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Requests per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 490 - }, - "id": 321, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|write|list|stat\"}[$__rate_interval]))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Requests per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Other Request P99 per Instance.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 3, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 490 - }, - "id": 322, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - }, - "pluginVersion": "12.0.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~\"read|write|list|Writer::write|Writer::close|Reader::read\"}[$__rate_interval])))", - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", - "range": true, - "refId": "A" - } - ], - "title": "Other Request P99 per Instance", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Total traffic as in bytes by instance and operation", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 497 - }, - "id": 323, - "options": { - "legend": { - "calcs": [ - "lastNotNull" + "mean" ], "displayMode": "table", "placement": "bottom", @@ -7019,26 +9970,49 @@ "sort": "none" } }, - "pluginVersion": "12.0.0", + "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg", + "range": true, + "refId": "C" } ], - "title": "Opendal traffic", + "title": "Flush Elapsed Time", "type": "timeseries" }, { @@ -7046,7 +10020,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "OpenDAL error counts per Instance.", + "description": "Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs.", "fieldConfig": { "defaults": { "color": { @@ -7059,6 +10033,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -7096,17 +10071,247 @@ "value": 80 } ] + }, + "unit": "Bps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*-files$" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 139 + }, + "id": 541, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-bytes", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-files", + "range": true, + "refId": "B" + } + ], + "title": "Flush Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing flush task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 497 + "h": 8, + "w": 8, + "x": 0, + "y": 147 }, - "id": 334, + "id": 499, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_flush_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Flush", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction OPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 147 + }, + "id": 491, "options": { "legend": { "calcs": [ @@ -7122,7 +10327,7 @@ "sort": "desc" } }, - "pluginVersion": "12.0.0", + "pluginVersion": "11.6.0", "targets": [ { "datasource": { @@ -7130,18 +10335,461 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!=\"NotFound\"}[$__rate_interval]))", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))", "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", + "legendFormat": "[{{ instance }}]-[{{pod}}]", "range": true, "refId": "A" } ], - "title": "OpenDAL errors per Instance", + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Ongoing compaction task count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 147 + }, + "id": 496, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_inflight_compaction_count", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]", + "range": true, + "refId": "A" + } + ], + "title": "Inflight Compaction", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 155 + }, + "id": 493, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Compaction P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction latency by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 155 + }, + "id": 492, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Elapsed Time per Instance by Stage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 155 + }, + "id": 500, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-input", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-output", + "range": true, + "refId": "B" + } + ], + "title": "Compaction Input/Output Bytes", "type": "timeseries" } ], - "title": "OpenDAL", + "title": "Flush and Compaction", "type": "row" }, { @@ -7150,16 +10798,16 @@ "h": 1, "w": 24, "x": 0, - "y": 28 + "y": 139 }, - "id": 351, + "id": 549, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, - "description": "Triggered region flush total", + "description": "Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates.", "fieldConfig": { "defaults": { "color": { @@ -7211,21 +10859,23 @@ } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 195 + "y": 164, + "w": 8, + "h": 6 }, - "id": 350, + "id": 542, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7243,14 +10893,28 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "meta_triggered_region_flush_total", + "expr": "histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", + "legendFormat": "{{type}}-p95", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-p99", + "range": true, + "refId": "B" } ], - "title": "Triggered region flush total", + "title": "Index Apply Elapsed Time", "type": "timeseries" }, { @@ -7258,7 +10922,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Triggered region checkpoint total", + "description": "Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays.", "fieldConfig": { "defaults": { "color": { @@ -7310,21 +10974,23 @@ } ] }, - "unit": "none" + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 195 + "x": 8, + "y": 164, + "w": 8, + "h": 6 }, - "id": 352, + "id": 543, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7342,14 +11008,28 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "meta_triggered_region_checkpoint_total", + "expr": "histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", + "legendFormat": "{{type}}-{{stage}}-p95", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-{{stage}}-p99", + "range": true, + "refId": "B" } ], - "title": "Triggered region checkpoint total", + "title": "Index Create Elapsed Time", "type": "timeseries" }, { @@ -7357,7 +11037,135 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Topic estimated max replay size", + "description": "Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*-bytes$" + }, + "properties": [ + { + "id": "unit", + "value": "Bps" + } + ] + } + ] + }, + "gridPos": { + "x": 16, + "y": 164, + "w": 8, + "h": 6 + }, + "id": 544, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-rows", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-bytes", + "range": true, + "refId": "B" + } + ], + "title": "Index Create Rows and Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work.", "fieldConfig": { "defaults": { "color": { @@ -7414,16 +11222,18 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 227 + "y": 170, + "w": 8, + "h": 6 }, - "id": 353, + "id": 545, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7441,14 +11251,28 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "meta_topic_estimated_replay_size", + "expr": "greptime_index_apply_memory_usage", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{topic_name}}", + "legendFormat": "apply", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (greptime_index_create_memory_usage)", + "hide": false, + "instant": false, + "legendFormat": "create-{{type}}", + "range": true, + "refId": "B" } ], - "title": "Topic estimated replay size", + "title": "Index Memory Usage", "type": "timeseries" }, { @@ -7456,7 +11280,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Kafka logstore's bytes traffic", + "description": "Index read and write byte rates by operation and file type for puffin and intermediate files.", "fieldConfig": { "defaults": { "color": { @@ -7508,21 +11332,23 @@ } ] }, - "unit": "bytes" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 227 + "x": 8, + "y": 170, + "w": 8, + "h": 6 }, - "id": 354, + "id": 546, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "mean" + ], + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -7540,18 +11366,249 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", + "expr": "sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))", + "hide": false, "instant": false, - "legendFormat": "{{pod}}-{{logstore}}", + "legendFormat": "{{type}}-{{file_type}}", "range": true, "refId": "A" } ], - "title": "Kafka logstore's bytes traffic", + "title": "Index IO Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Index IO operation rates by operation and file type, including read, write, seek, and flush operations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 170, + "w": 8, + "h": 6 + }, + "id": 547, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{type}}-{{file_type}}", + "range": true, + "refId": "A" + } + ], + "title": "Index IO Operations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Index-related cache hits, misses, and evictions from Mito caches.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 176, + "w": 8, + "h": 6 + }, + "id": 548, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "hit-{{type}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "miss-{{type}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "evict-{{type}}-{{cause}}", + "range": true, + "refId": "C" + } + ], + "title": "Index Cache", "type": "timeseries" } ], - "title": "Remote WAL", + "title": "Index", "type": "row" }, { @@ -7560,10 +11617,292 @@ "h": 1, "w": 24, "x": 0, - "y": 29 + "y": 140 }, - "id": 324, + "id": 451, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 165, + "w": 8, + "h": 8 + }, + "id": 436, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(greptime_meta_inactive_regions)", + "hide": false, + "instant": false, + "legendFormat": "inactive-regions", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(greptime_lease_expired_region)", + "hide": false, + "instant": false, + "legendFormat": "lease-expired", + "range": true, + "refId": "B" + } + ], + "title": "Inactive and Lease-expired Regions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 165, + "w": 8, + "h": 8 + }, + "id": 437, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "meta-recv-rate", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(greptime_meta_heartbeat_connection_num)", + "hide": false, + "instant": false, + "legendFormat": "meta-connections", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "frontend-send", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "frontend-recv", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "datanode-send", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "datanode-recv", + "range": true, + "refId": "F" + } + ], + "title": "Heartbeat Health", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -7602,12 +11941,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 196 + "x": 16, + "y": 165, + "w": 8, + "h": 8 }, - "id": 325, + "id": 525, "options": { "colWidth": 0.9, "legend": { @@ -7716,12 +12055,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 196 + "x": 0, + "y": 173, + "w": 8, + "h": 8 }, - "id": 326, + "id": 526, "options": { "legend": { "calcs": [], @@ -7743,7 +12082,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "greptime_meta_region_migration_error", + "expr": "rate(greptime_meta_region_migration_error[$__rate_interval])", "instant": false, "legendFormat": "{{pod}}-{{state}}-{{error_type}}", "range": true, @@ -7815,12 +12154,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 228 + "x": 8, + "y": 173, + "w": 8, + "h": 8 }, - "id": 327, + "id": 527, "options": { "legend": { "calcs": [], @@ -7914,12 +12253,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 228 + "x": 16, + "y": 173, + "w": 8, + "h": 8 }, - "id": 339, + "id": 528, "options": { "legend": { "calcs": [], @@ -8013,12 +12352,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 236 + "y": 181, + "w": 8, + "h": 8 }, - "id": 340, + "id": 529, "options": { "legend": { "calcs": [], @@ -8049,6 +12388,19 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg", + "range": true, + "refId": "B" } ], "title": "SQL Execution Latency (RDS)", @@ -8116,12 +12468,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 236 + "x": 8, + "y": 181, + "w": 8, + "h": 8 }, - "id": 341, + "id": 530, "options": { "legend": { "calcs": [], @@ -8152,6 +12504,19 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{pod}}]-[{{name}}]-avg", + "range": true, + "refId": "B" } ], "title": "Handler Execution Latency", @@ -8219,12 +12584,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 244 + "x": 16, + "y": 181, + "w": 8, + "h": 8 }, - "id": 342, + "id": 531, "options": { "legend": { "calcs": [], @@ -8247,7 +12612,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", + "expr": "histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -8265,7 +12630,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "Rate of heartbeats received by metasrv from datanodes and frontends.", "fieldConfig": { "defaults": { "color": { @@ -8322,12 +12687,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 244 + "x": 0, + "y": 189, + "w": 8, + "h": 8 }, - "id": 345, + "id": 532, "options": { "legend": { "calcs": [], @@ -8368,7 +12733,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "p99 and average latency of metasrv key-value store operations by op and target.", "fieldConfig": { "defaults": { "color": { @@ -8425,12 +12790,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 252 + "x": 8, + "y": 189, + "w": 8, + "h": 8 }, - "id": 343, + "id": 533, "options": { "legend": { "calcs": [], @@ -8453,7 +12818,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", + "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -8461,6 +12826,19 @@ "range": true, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{pod}}]-[{{op}}]-[{{target}}]-avg", + "range": true, + "refId": "B" } ], "title": "Meta KV Ops Latency", @@ -8471,7 +12849,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "Rate of metasrv key-value store operations by op.", "fieldConfig": { "defaults": { "color": { @@ -8528,12 +12906,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 252 + "x": 16, + "y": 189, + "w": 8, + "h": 8 }, - "id": 346, + "id": 534, "options": { "legend": { "calcs": [], @@ -8574,7 +12952,7 @@ "type": "prometheus", "uid": "${metrics}" }, - "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", + "description": "p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step.", "fieldConfig": { "defaults": { "color": { @@ -8631,12 +13009,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, "x": 0, - "y": 260 + "y": 197, + "w": 8, + "h": 8 }, - "id": 347, + "id": 535, "options": { "legend": { "calcs": [], @@ -8659,7 +13037,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -8675,7 +13053,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8692,7 +13070,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8709,7 +13087,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8726,7 +13104,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8743,7 +13121,7 @@ }, "disableTextWrap": false, "editorMode": "code", - "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", + "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -8814,17 +13192,17 @@ } ] }, - "unit": "s" + "unit": "ops" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 260 + "x": 8, + "y": 197, + "w": 8, + "h": 8 }, - "id": 355, + "id": 536, "options": { "legend": { "calcs": [], @@ -8846,7 +13224,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "greptime_meta_reconciliation_stats", + "expr": "rate(greptime_meta_reconciliation_stats[$__rate_interval])", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{table_type}}-{{type}}", @@ -8919,12 +13297,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 268 + "x": 16, + "y": 197, + "w": 8, + "h": 8 }, - "id": 356, + "id": 537, "options": { "legend": { "calcs": [], @@ -8946,7 +13324,7 @@ "uid": "${metrics}" }, "editorMode": "code", - "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", + "expr": "histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "{{procedure_name}}-{{step}}-P90", @@ -8967,9 +13345,3174 @@ "h": 1, "w": 24, "x": 0, - "y": 30 + "y": 141 }, - "id": 328, + "id": 553, + "panels": [ + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto", + "wrapText": false + }, + "inspect": false + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "disk_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 166, + "w": 24, + "h": 18 + }, + "id": 438, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": true, + "fields": [ + "region_rows", + "disk_size" + ], + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Hotspot Regions", + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write load of each datanode over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 184, + "w": 8, + "h": 11 + }, + "id": 439, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "greptime_datanode_history_load", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Distribution of write load across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "min": 0, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 184, + "w": 8, + "h": 11 + }, + "id": 440, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "greptime_datanode_history_load", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "datanode-{{datanode_id}}({{instance}})", + "range": false, + "refId": "A", + "useBackend": false + } + ], + "title": "Datanode Load(Write) Distribution", + "type": "piechart" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "Distribution of leader regions and data size across datanodes.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "fieldMinMax": false, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 184, + "w": 8, + "h": 11 + }, + "id": 441, + "options": { + "displayLabels": [ + "value" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^data_size$/", + "values": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Datanode Data Distribution", + "type": "piechart" + } + ], + "title": "Hotspot", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 554, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer action count", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 167, + "w": 8, + "h": 8 + }, + "id": 442, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)", + "hide": false, + "instant": false, + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Actions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer gate stop count by gate and reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 167, + "w": 8, + "h": 8 + }, + "id": 443, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{gate}} / {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Gate Stops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer datanode count by state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 167, + "w": 8, + "h": 8 + }, + "id": 444, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (state) (greptime_region_balancer_datanodes)", + "hide": false, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Datanodes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer region count by state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 175, + "w": 8, + "h": 8 + }, + "id": 445, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (state) (greptime_region_balancer_regions)", + "hide": false, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Regions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Region balancer datanode stability statistics by state", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*count$" + }, + "properties": [ + { + "id": "unit", + "value": "short" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": ".*ratio$" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "x": 8, + "y": 175, + "w": 8, + "h": 8 + }, + "id": 446, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "builder", + "expr": "sum by (state) (greptime_region_balancer_datanode_stability)", + "hide": false, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Region Balancer Datanode Stability", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition action count by result", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": -1, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 175, + "w": 8, + "h": 8 + }, + "id": 447, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{result}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Actions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition gate stop count by gate and reason", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 183, + "w": 8, + "h": 8 + }, + "id": 448, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "{{gate}} / {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Gate Stops", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition sampling elapsed time by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 183, + "w": 8, + "h": 8 + }, + "id": 449, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Sampling P99", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Auto repartition executor elapsed time by stage", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 183, + "w": 8, + "h": 8 + }, + "id": 450, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "{{stage}}", + "range": true, + "refId": "A" + } + ], + "title": "Auto Repartition Executor P99", + "type": "timeseries" + } + ], + "title": "Autopilot", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 143 + }, + "id": 452, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 168 + }, + "id": 505, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 178 + }, + "id": 506, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"read|Reader::read\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Read QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Read P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 178 + }, + "id": 507, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~\"read|Reader::read\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~\"read|Reader::read\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"read|Reader::read\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Read P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 178 + }, + "id": 508, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Write QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 185 + }, + "id": 509, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Write P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List QPS per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 185 + }, + "id": 510, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "List P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 185 + }, + "id": 511, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation=\"list\"}[$__rate_interval])))", + "instant": false, + "interval": "", + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=\"list\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=\"list\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "List P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Requests per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 192 + }, + "id": 512, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Other Request P99 and average per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 192 + }, + "id": 513, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg", + "range": true, + "refId": "B" + } + ], + "title": "Other Request P99 and Avg per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Total traffic as in bytes by instance and operation", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 192 + }, + "id": 514, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval]))", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Opendal traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "OpenDAL error counts per Instance.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 199 + }, + "id": 515, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!=\"NotFound\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", + "range": true, + "refId": "A" + } + ], + "title": "OpenDAL errors per Instance", + "type": "timeseries" + } + ], + "title": "Object Store", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 144 + }, + "id": 552, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 169 + }, + "id": 494, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", + "range": true, + "refId": "C" + } + ], + "title": "WAL write size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Raft engine (local disk) log store sync latency, p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 169 + }, + "id": 497, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "WAL sync duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Write-ahead log operations latency at p99", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 169 + }, + "id": 498, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Log Store op duration seconds", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region flush total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 177 + }, + "id": 516, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_flush_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region flush total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Triggered region checkpoint total", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 177 + }, + "id": 517, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_triggered_region_checkpoint_total", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Triggered region checkpoint total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Topic estimated max replay size", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 177 + }, + "id": 518, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "meta_topic_estimated_replay_size", + "instant": false, + "legendFormat": "{{pod}}-{{topic_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Topic estimated replay size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Kafka logstore's bytes traffic", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 185 + }, + "id": 519, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", + "instant": false, + "legendFormat": "{{pod}}-{{logstore}}", + "range": true, + "refId": "A" + } + ], + "title": "Kafka logstore's bytes traffic", + "type": "timeseries" + } + ], + "title": "WAL", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 145 + }, + "id": 453, "panels": [ { "datasource": { @@ -9032,11 +16575,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 3282 + "y": 170 }, - "id": 329, + "id": 520, "options": { "legend": { "calcs": [], @@ -9127,11 +16670,11 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 3282 + "w": 8, + "x": 8, + "y": 170 }, - "id": 330, + "id": 521, "options": { "legend": { "calcs": [], @@ -9169,6 +16712,19 @@ "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "[{{instance}}]-[{{pod}}]-avg", + "range": true, + "refId": "C" } ], "title": "Flow Ingest Latency", @@ -9235,11 +16791,11 @@ }, "gridPos": { "h": 8, - "w": 9, - "x": 0, - "y": 3290 + "w": 8, + "x": 16, + "y": 170 }, - "id": 331, + "id": 522, "options": { "legend": { "calcs": [], @@ -9343,11 +16899,11 @@ }, "gridPos": { "h": 8, - "w": 9, - "x": 9, - "y": 3290 + "w": 8, + "x": 0, + "y": 178 }, - "id": 332, + "id": 523, "options": { "legend": { "calcs": [], @@ -9438,11 +16994,11 @@ }, "gridPos": { "h": 8, - "w": 6, - "x": 18, - "y": 3290 + "w": 8, + "x": 8, + "y": 178 }, - "id": 333, + "id": 524, "options": { "legend": { "calcs": [], @@ -9477,2077 +17033,793 @@ "type": "row" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 31 + "y": 146 }, - "id": 357, - "panels": [], - "title": "Trigger", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "bf9fzta69bhtsa" - }, - "description": "Total number of triggers currently defined.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "id": 461, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" + "description": "Total number of triggers currently defined.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 32 - }, - "id": 358, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "greptime_trigger_count{}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Trigger Count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 359, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-p75", - "range": true, - "refId": "B" - } - ], - "title": "Trigger Eval Elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Rate of failed trigger evaluations.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 360, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Trigger Eval Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed time to send trigger alerts to notification channels.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 361, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75", - "range": true, - "refId": "B" - } - ], - "title": "Send Alert Elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Rate of failures when sending trigger alerts.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 364, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Send Alert Failure Rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Elapsed time to persist trigger alert records.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 363, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75", - "range": true, - "refId": "B" - } - ], - "title": "Save Alert Elapsed", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Rate of failures when persisting trigger alert records.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 362, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])", - "hide": false, - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Save Alert Failure Rate", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 64 - }, - "id": 366, - "panels": [], - "title": "Hotspot", - "type": "row" - }, - { - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto", - "wrapText": false - }, - "inspect": false - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "disk_size" - }, - "properties": [ - { - "id": "unit", - "value": "bytes" - } - ] - } - ] - }, - "gridPos": { - "h": 18, - "w": 24, - "x": 0, - "y": 65 - }, - "id": 365, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "enablePagination": true, - "fields": [ - "region_rows", - "disk_size" - ], - "reducer": [ - "sum" - ], - "show": true - }, - "showHeader": true, - "sortBy": [] - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Hotspot Regions", - "type": "table" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Write load of each datanode over time.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "binBps" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "total_region_rows", - "total_written_bytes_since_open", - "total_disk_size", - "total_sst_size", - "total_sst_num", - "total_memtable_size", - "total_manifest_size", - "total_index_size", - "cluster_disk_ratio_percent", - "cluster_sst_ratio_percent", - "cluster_rows_ratio_percent", - "max_region_disk_size", - "max_region_rows", - "leader_region_count", - "disk_ratio_percent", - "data_size" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [] - } - ] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 83 - }, - "id": 368, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "greptime_datanode_history_load", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "datanode-{{datanode_id}}({{instance}})", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Datanode Load(Write)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Distribution of write load across datanodes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "fieldMinMax": false, - "mappings": [], - "min": 0, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 12, - "y": 83 - }, - "id": 369, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "greptime_datanode_history_load", - "format": "time_series", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "datanode-{{datanode_id}}({{instance}})", - "range": false, - "refId": "A", - "useBackend": false - } - ], - "title": "Datanode Load(Write) Distribution", - "type": "piechart" - }, - { - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "description": "Distribution of leader regions and data size across datanodes.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "fieldMinMax": false, - "mappings": [], - "unit": "bytes" - }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "total_region_rows", - "total_written_bytes_since_open", - "total_disk_size", - "total_sst_size", - "total_sst_num", - "total_memtable_size", - "total_manifest_size", - "total_index_size", - "cluster_disk_ratio_percent", - "cluster_sst_ratio_percent", - "cluster_rows_ratio_percent", - "max_region_disk_size", - "max_region_rows", - "leader_region_count", - "disk_ratio_percent", - "data_size" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, - "viz": true - } - } - ] - } - ] - }, - "gridPos": { - "h": 11, - "w": 12, - "x": 0, - "y": 94 - }, - "id": 367, - "options": { - "displayLabels": [ - "value" - ], - "legend": { - "displayMode": "list", - "placement": "right", - "showLegend": true, - "values": [ - "percent" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/^data_size$/", - "values": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "dataset": "information_schema", - "datasource": { - "type": "mysql", - "uid": "${information_schema}" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" + "viz": false }, - "type": "groupBy" + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] } - ], - "limit": 50 - } + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 171, + "w": 8, + "h": 6 + }, + "id": 454, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_trigger_count", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Trigger Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 171, + "w": 8, + "h": 6 + }, + "id": 455, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "C" + } + ], + "title": "Trigger Eval Elapsed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Rate of failed trigger evaluations.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 171, + "w": 8, + "h": 6 + }, + "id": 456, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Trigger Eval Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed time to send trigger alerts to notification channels.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 177, + "w": 8, + "h": 6 + }, + "id": 457, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "C" + } + ], + "title": "Send Alert Elapsed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Rate of failures when sending trigger alerts.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 8, + "y": 177, + "w": 8, + "h": 6 + }, + "id": 458, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Send Alert Failure Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Elapsed time to persist trigger alert records.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 16, + "y": 177, + "w": 8, + "h": 6 + }, + "id": 459, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p99", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "p75", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "avg", + "range": true, + "refId": "C" + } + ], + "title": "Save Alert Elapsed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "Rate of failures when persisting trigger alert records.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 183, + "w": 8, + "h": 6 + }, + "id": 460, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Save Alert Failure Rate", + "type": "timeseries" } ], - "title": "Datanode Data Distribution", - "type": "piechart" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 105 - }, - "id": 370, - "panels": [], - "title": "Autopilot", + "title": "Trigger", "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer action count", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 106 - }, - "id": 371, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)", - "hide": false, - "instant": false, - "legendFormat": "{{result}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Actions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer gate stop count by gate and reason", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 106 - }, - "id": 372, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{gate}} / {{reason}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Gate Stops", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer datanode count by state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 114 - }, - "id": 373, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (state) (greptime_region_balancer_datanodes)", - "hide": false, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Datanodes", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer region count by state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 114 - }, - "id": 374, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (state) (greptime_region_balancer_regions)", - "hide": false, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Regions", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Region balancer datanode stability statistics by state", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "binBps" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*count$" - }, - "properties": [ - { - "id": "unit", - "value": "short" - } - ] - }, - { - "matcher": { - "id": "byRegexp", - "options": ".*ratio$" - }, - "properties": [ - { - "id": "unit", - "value": "percentunit" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 122 - }, - "id": 375, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "builder", - "expr": "sum by (state) (greptime_region_balancer_datanode_stability)", - "hide": false, - "instant": false, - "legendFormat": "{{state}}", - "range": true, - "refId": "A" - } - ], - "title": "Region Balancer Datanode Stability", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition action count by result", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": -1, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 130 - }, - "id": 376, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{result}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Actions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition gate stop count by gate and reason", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 130 - }, - "id": 377, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))", - "hide": false, - "instant": false, - "legendFormat": "{{gate}} / {{reason}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Gate Stops", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition sampling elapsed time by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 138 - }, - "id": 378, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{stage}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Sampling P99", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "description": "Auto repartition executor elapsed time by stage", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 138 - }, - "id": 379, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "11.6.0", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${metrics}" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))", - "hide": false, - "instant": false, - "legendFormat": "{{stage}}", - "range": true, - "refId": "A" - } - ], - "title": "Auto Repartition Executor P99", - "type": "timeseries" } ], "preload": false, @@ -11686,7 +17958,7 @@ }, "timepicker": {}, "timezone": "", - "title": "GreptimeDB Copy", + "title": "GreptimeDB", "uid": "dflfbxbwvvchsa", - "version": 67 -} \ No newline at end of file + "version": 68 +} diff --git a/grafana/dashboards/metrics/standalone/dashboard.md b/grafana/dashboards/metrics/standalone/dashboard.md index 672a2a83641..a3a894cffa4 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.md +++ b/grafana/dashboards/metrics/standalone/dashboard.md @@ -4,144 +4,154 @@ | Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` | | Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- | | Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` | -| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- | -| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- | +| Total Query Rate | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `stat` | Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. | `prometheus` | `reqps` | `queries` | +| User-facing Error Rate | `sum(rate(greptime_servers_error[$__rate_interval]))` | `stat` | Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. | `prometheus` | `eps` | `errors` | +| Recent Restarts | `sum(changes(process_start_time_seconds[$__range]))` | `stat` | Process restarts over the selected time range across GreptimeDB roles. | `prometheus` | `short` | `restarts` | | Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- | | Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- | +| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- | +| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- | | Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- | +| Total Ingestion Rate Trend | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `timeseries` | Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. | `prometheus` | `rowsps` | `ingestion` | +| Total Query Rate Trend | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. | `prometheus` | `reqps` | `queries` | +| HTTP Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))`
`sum(rate(greptime_servers_http_requests_elapsed_sum{path!~"/health\|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. | `prometheus` | `s` | `http-p99` | +| gRPC Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | Tail and average latency for gRPC requests served by frontends. | `prometheus` | `s` | `grpc-p99` | # Ingestion | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` | -| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `http-logs` | -# Queries +| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `timeseries` | Total ingestion rate.

Here we listed 3 primary protocols:

- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` | +| Ingestion Rate by Protocol | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval]))` | `timeseries` | Rows, samples, or documents ingested by primary observability and table-ingestion protocols. | `prometheus` | `rowsps` | `table-operator` | +| Ingestion Latency by Protocol | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. | `prometheus` | `s` | `prometheus-write` | +| Bulk Insert Message Rows and Size | `sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval]))`
`sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval]))` | `timeseries` | Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. | `prometheus` | `rowsps` | `rows` | +| Prom Store Flush Pipeline | `sum(rate(greptime_prom_store_flush_total[$__rate_interval]))`
`sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval]))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval])))` | `timeseries` | Remote-write pending-row flush operations, flushed rows, and p99 flush latency. | `prometheus` | `short` | `flush-ops` | +| OTLP Trace Failures | `sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval]))` | `timeseries` | OTLP trace ingestion failures reported by frontends. | `prometheus` | `eps` | `trace-failures` | +# Health | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.

Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API

Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` | +| Protocol Error Rates | `sum by (protocol) (rate(greptime_servers_error[$__rate_interval]))`
`sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval]))`
`sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics",code!~"2.."}[$__rate_interval]))`
`sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~"0\|OK"}[$__rate_interval]))` | `timeseries` | User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. | `prometheus` | `eps` | `server-{{protocol}}` | +| Frontend and Query Rejections | `sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval]))`
`sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))` | `timeseries` | Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. | `prometheus` | `rps` | `request-memory` | +| Datanode Write Failures | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))` | `timeseries` | Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. | `prometheus` | `eps` | `region-request-[{{instance}}]-[{{pod}}]` | +| Buffered Ingestion Loss | `sum(rate(greptime_pending_rows_flush_failures[$__rate_interval]))`
`sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval]))` | `timeseries` | Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. | `prometheus` | `eps` | `flush-failures` | +| Mito Backpressure and Failures | `sum(rate(greptime_mito_write_reject_total[$__rate_interval]))`
`sum(rate(greptime_mito_write_stall_total[$__rate_interval]))`
`sum(rate(greptime_mito_flush_failure_total[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_failure_total[$__rate_interval]))` | `timeseries` | Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. | `prometheus` | `eps` | `write-reject` | +| Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))`
`sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` | +| OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` | +| Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`
`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` | +| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`
`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` | +| Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` | +# Capacity +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Runtime Threads | `sum by (instance, pod) (greptime_runtime_threads_alive)`
`sum by (instance, pod) (greptime_runtime_threads_idle)` | `timeseries` | Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. | `prometheus` | `short` | `alive-[{{instance}}]-[{{pod}}]` | +| Request Memory Utilization | `sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes)` | `timeseries` | Frontend request memory usage divided by configured request memory limit. | `prometheus` | `percentunit` | `[{{instance}}]-[{{pod}}]` | +| Query Memory Usage | `sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)` | `timeseries` | Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]` | +| Scan and Compaction Memory | `sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes)`
`sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes)`
`sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes)` | `timeseries` | Datanode scan memory usage and compaction memory utilization. | `prometheus` | `bytes` | `scan-[{{instance}}]-[{{pod}}]` | +| Write Buffer and Active Stalling | `sum by (instance, pod) (greptime_mito_write_buffer_bytes)`
`sum by (instance, pod) (greptime_mito_write_stalling_count)` | `timeseries` | Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. | `prometheus` | `bytes` | `buffer-[{{instance}}]-[{{pod}}]` | +| Prom Store Backlog | `sum by (instance, pod) (greptime_prom_store_pending_rows)`
`sum by (instance, pod) (greptime_prom_store_pending_batches)`
`sum by (instance, pod) (greptime_prom_store_pending_workers)` | `timeseries` | Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. | `prometheus` | `short` | `rows-[{{instance}}]-[{{pod}}]` | +| Inflight Flush and Compaction | `sum by (instance, pod) (greptime_mito_inflight_flush_count)`
`sum by (instance, pod) (greptime_mito_inflight_compaction_count)` | `timeseries` | Current in-flight flush and compaction tasks on datanodes. | `prometheus` | `short` | `flush-[{{instance}}]-[{{pod}}]` | # Resources | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Datanode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` | -| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | -| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` | -| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` | -| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | -| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` | +| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Frontend Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | +| Datanode Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` | +| Metasrv Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` | +| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` | +| Flownode Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` | +# Queries +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Query Rate by Protocol | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | Query API call rates by protocol, collected from frontends. | `prometheus` | `reqps` | `mysql` | +| Query Latency by Protocol | `histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | p95, p99, and average query latency by main frontend protocol. | `prometheus` | `s` | `mysql-p95` | +| Query Stage Latency | `histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))` | `timeseries` | p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. | `prometheus` | `s` | `p95-{{stage}}` | +| Merge Scan Fan-out and Errors | `sum by (instance, pod) (greptime_query_merge_scan_regions)`
`sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))` | `timeseries` | Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. | `prometheus` | `short` | `regions-[{{instance}}]-[{{pod}}]` | +| Pushdown Fallback Errors | `sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))` | `timeseries` | Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. | `prometheus` | `eps` | `pushdown-fallback-errors` | +| PromQL Series Count | `sum by (instance, pod) (greptime_promql_series_count)` | `timeseries` | Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]` | +| Connections and Prepared Statements | `sum by (instance, pod) (greptime_servers_mysql_connection_count)`
`sum by (instance, pod) (greptime_servers_postgres_connection_count)`
`sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval]))` | `timeseries` | MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. | `prometheus` | `short` | `mysql-connections-[{{instance}}]-[{{pod}}]` | # Frontend Requests | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | | HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` | -| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | -| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | -| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | -| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | -| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` | -| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | -| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| HTTP P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))`
`sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~"/health\|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` | +| gRPC P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | gRPC P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` | +| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | +| MySQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))` | `timeseries` | MySQL P99 and average per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` | +| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` | +| PostgreSQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))` | `timeseries` | PostgreSQL P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | # Frontend to Datanode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | -| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | -| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | -| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` | -# Mito Engine +| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Region Call P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval])))`
`sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))` | `timeseries` | Region Call P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` | +| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` | +# Datanode | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | -| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | -| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | -| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | -| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | -| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | -| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | -| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | -| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` | -| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` | -| Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | -| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | -| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | -| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`
`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` | +| Region Request Failures and Failed Inserts | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))` | `timeseries` | Datanode region request failures and failed inserts by instance. | `prometheus` | `eps` | `request-fail-[{{instance}}]-[{{pod}}]` | +| Write Rejects and Stalls | `sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval]))`
`sum by (instance, pod) (greptime_mito_write_stalling_count)` | `timeseries` | Mito write rejects, write stall events, and active write stalling by datanode. | `prometheus` | `short` | `reject-[{{instance}}]-[{{pod}}]` | +| Flush and Compaction Failures | `sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval]))` | `timeseries` | Mito flush and compaction failure rates by datanode. | `prometheus` | `eps` | `flush-[{{instance}}]-[{{pod}}]` | +| Mito GC Health | `sum(rate(greptime_mito_gc_runs_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. | `prometheus` | `short` | `runs` | +| Mito GC Duration | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval])))`
`sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval]))` | `timeseries` | P99 and average Mito garbage-collection duration by stage. | `prometheus` | `s` | `{{stage}}-p99` | +# Storage +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))` | `timeseries` | Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +| Request Wait P99 and Avg per Worker | `histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))`
`sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval]))` | `timeseries` | Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{worker}}]-p95` | +| Write Buffer per Instance | `greptime_mito_write_buffer_bytes` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` | +| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` | +| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` | +| Read Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Read Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Write Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Write Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` | +| Cached Bytes per Instance | `greptime_mito_cache_bytes` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` | | Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`
`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` | -| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` | +| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Active series and field-builder counts per memtable by instance. | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` | | Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`
`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` | -| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` | -# OpenDAL +| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` | +# Flush and Compaction | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | -| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | -| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` | -# Remote WAL +| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` | +| Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`
`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` | +| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` | +| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` | +| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` | +| Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`
`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` | +# Index | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | -| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | -| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | -| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` | -| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` | +| Index Apply Elapsed Time | `histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. | `prometheus` | `s` | `{{type}}-p95` | +| Index Create Elapsed Time | `histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. | `prometheus` | `s` | `{{type}}-{{stage}}-p95` | +| Index Create Rows and Bytes | `sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))`
`sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))` | `timeseries` | Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. | `prometheus` | `rowsps` | `{{type}}-rows` | +| Index Memory Usage | `greptime_index_apply_memory_usage`
`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` | +| Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` | +| Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` | +| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` | # Metasrv | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | +| Inactive and Lease-expired Regions | `sum(greptime_meta_inactive_regions)`
`sum(greptime_lease_expired_region)` | `timeseries` | Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. | `prometheus` | `short` | `inactive-regions` | +| Heartbeat Health | `sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))`
`sum(greptime_meta_heartbeat_connection_num)`
`sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval]))`
`sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval]))` | `timeseries` | Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. | `prometheus` | `short` | `meta-recv-rate` | | Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` | -| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` | +| Region migration error | `rate(greptime_meta_region_migration_error[$__rate_interval])` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` | | Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` | | Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` | -| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` | +| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))`
`sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` | | Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) ( rate(greptime_meta_handler_execute_bucket[$__rate_interval]) -))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` | -| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` | -| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` | -| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` | -| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` | -| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` | -| Reconciliation stats | `greptime_meta_reconciliation_stats` | `timeseries` | Reconciliation stats | `prometheus` | `s` | `{{pod}}-{{table_type}}-{{type}}` | -| Reconciliation steps | `histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` | -# Flownode -| Title | Query | Type | Description | Datasource | Unit | Legend Format | -| --- | --- | --- | --- | --- | --- | --- | -| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | -| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` | -| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | -| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | -| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` | -# Trigger -| Title | Query | Type | Description | Datasource | Unit | Legend Format | -| --- | --- | --- | --- | --- | --- | --- | -| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` | -| Trigger Eval Elapsed | `histogram_quantile(0.99, - rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]) -)`
`histogram_quantile(0.75, - rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]) -)` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | -| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` | -| Send Alert Elapsed | `histogram_quantile(0.99, - rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]) -)`
`histogram_quantile(0.75, - rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]) -)` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99` | -| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` | -| Save Alert Elapsed | `histogram_quantile(0.99, - rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]) -)`
`histogram_quantile(0.75, - rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]) -)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` | -| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` | +))`
`sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` | +| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` | +| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Rate of heartbeats received by metasrv from datanodes and frontends. | `prometheus` | `s` | `{{pod}}` | +| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))`
`sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average latency of metasrv key-value store operations by op and target. | `prometheus` | `s` | `{{pod}}-{{op}} p99` | +| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Rate of metasrv key-value store operations by op. | `prometheus` | `none` | `{{pod}}-{{op}} p99` | +| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))` | `timeseries` | p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` | +| Reconciliation stats | `rate(greptime_meta_reconciliation_stats[$__rate_interval])` | `timeseries` | Reconciliation stats | `prometheus` | `ops` | `{{pod}}-{{table_type}}-{{type}}` | +| Reconciliation steps | `histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` | # Hotspot | Title | Query | Type | Description | Datasource | Unit | Legend Format | | --- | --- | --- | --- | --- | --- | --- | @@ -240,3 +250,45 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data | Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` | | Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` | | Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` | +# Object Store +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Read P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"read\|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Write P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| List P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="list"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))` | `timeseries` | List P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` | +| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Other Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` | +| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` | +# WAL +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` | +| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` | +| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` | +| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | +| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` | +| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` | +| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` | +# Flownode +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` | +| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` | +| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` | +| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` | +| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` | +# Trigger +| Title | Query | Type | Description | Datasource | Unit | Legend Format | +| --- | --- | --- | --- | --- | --- | --- | +| Trigger Count | `greptime_trigger_count` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` | +| Trigger Eval Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `p99` | +| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` | +| Send Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `p99` | +| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` | +| Save Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `p99` | +| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` | diff --git a/grafana/dashboards/metrics/standalone/dashboard.yaml b/grafana/dashboards/metrics/standalone/dashboard.yaml index c23501c0740..0ebad934ba6 100644 --- a/grafana/dashboards/metrics/standalone/dashboard.yaml +++ b/grafana/dashboards/metrics/standalone/dashboard.yaml @@ -29,24 +29,36 @@ groups: type: prometheus uid: ${metrics} legendFormat: __auto - - title: Total Storage Size + - title: Total Query Rate type: stat - description: Total number of data file size. - unit: decbytes + description: Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. + unit: reqps queries: - - expr: select SUM(disk_size) from information_schema.region_statistics; + - expr: sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval])) datasource: - type: mysql - uid: ${information_schema} - - title: Total Rows + type: prometheus + uid: ${metrics} + legendFormat: queries + - title: User-facing Error Rate type: stat - description: Total number of data rows in the cluster. Calculated by sum of rows from each region. - unit: sishort + description: Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. + unit: eps queries: - - expr: select SUM(region_rows) from information_schema.region_statistics; + - expr: sum(rate(greptime_servers_error[$__rate_interval])) datasource: - type: mysql - uid: ${information_schema} + type: prometheus + uid: ${metrics} + legendFormat: errors + - title: Recent Restarts + type: stat + description: Process restarts over the selected time range across GreptimeDB roles. + unit: short + queries: + - expr: sum(changes(process_start_time_seconds[$__range])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: restarts - title: Deployment type: stat description: The deployment topology of GreptimeDB. @@ -87,6 +99,24 @@ groups: datasource: type: mysql uid: ${information_schema} + - title: Total Storage Size + type: stat + description: Total number of data file size. + unit: decbytes + queries: + - expr: select SUM(disk_size) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} + - title: Total Rows + type: stat + description: Total number of data rows in the cluster. Calculated by sum of rows from each region. + unit: sishort + queries: + - expr: select SUM(region_rows) from information_schema.region_statistics; + datasource: + type: mysql + uid: ${information_schema} - title: Data Size type: stat description: The data size of wal/index/manifest in the GreptimeDB. @@ -104,6 +134,56 @@ groups: datasource: type: mysql uid: ${information_schema} + - title: Total Ingestion Rate Trend + type: timeseries + description: Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: ingestion + - title: Total Query Rate Trend + type: timeseries + description: Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. + unit: reqps + queries: + - expr: sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: queries + - title: HTTP Request P99 and Avg + type: timeseries + description: Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health|/metrics"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-p99 + - expr: sum(rate(greptime_servers_http_requests_elapsed_sum{path!~"/health|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-avg + - title: gRPC Request P99 and Avg + type: timeseries + description: Tail and average latency for gRPC requests served by frontends. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-p99 + - expr: sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-avg - title: Ingestion panels: - title: Total Ingestion Rate @@ -118,76 +198,483 @@ groups: - Log ingestion http API unit: rowsps queries: - - expr: sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval])) + - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: ingestion - - title: Ingestion Rate by Type + - title: Ingestion Rate by Protocol type: timeseries - description: | - Total ingestion rate. - - Here we listed 3 primary protocols: - - - Prometheus remote write - - Greptime's gRPC API (when using our ingest SDK) - - Log ingestion http API + description: Rows, samples, or documents ingested by primary observability and table-ingestion protocols. unit: rowsps queries: - - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval])) + - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: http-logs + legendFormat: table-operator - expr: sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: prometheus-remote-write - - title: Queries - panels: - - title: Total Query Rate + - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-logs + - expr: sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: loki-logs + - expr: sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: elasticsearch-docs + - expr: sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-metrics + - expr: sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-logs + - expr: sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-traces + - title: Ingestion Latency by Protocol type: timeseries - description: |- - Total rate of query API calls by protocol. This metric is collected from frontends. - - Here we listed 3 main protocols: - - MySQL - - Postgres - - Prometheus API - - Note that there are some other minor query APIs like /sql are not included - unit: reqps + description: p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. + unit: s queries: - - expr: sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval])) + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: mysql - - expr: sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval])) + legendFormat: prometheus-write + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: pg - - expr: sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval])) + legendFormat: http-logs + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: promql - - title: Resources - panels: - - title: Datanode Memory per Instance + legendFormat: loki-logs + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: elasticsearch + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-metrics + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-logs + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-traces + - expr: sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: prom-write-avg + - expr: sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-logs-avg + - expr: sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: loki-logs-avg + - expr: sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: elasticsearch-avg + - expr: sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-metrics-avg + - expr: sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-logs-avg + - expr: sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: otlp-traces-avg + - title: Bulk Insert Message Rows and Size type: timeseries - description: Current memory usage by instance + description: Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. + unit: rowsps + queries: + - expr: sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: rows + - expr: sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: bytes + - title: Prom Store Flush Pipeline + type: timeseries + description: Remote-write pending-row flush operations, flushed rows, and p99 flush latency. + unit: short + queries: + - expr: sum(rate(greptime_prom_store_flush_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-ops + - expr: sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-rows + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-p99 + - title: OTLP Trace Failures + type: timeseries + description: OTLP trace ingestion failures reported by frontends. + unit: eps + queries: + - expr: sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trace-failures + - title: Health + panels: + - title: Protocol Error Rates + type: timeseries + description: User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. + unit: eps + queries: + - expr: sum by (protocol) (rate(greptime_servers_error[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: server-{{protocol}} + - expr: sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: auth-{{code}} + - expr: sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics",code!~"2.."}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-{{method}}-{{path}}-{{code}} + - expr: sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~"0|OK"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-{{path}}-{{code}} + - title: Frontend and Query Rejections + type: timeseries + description: Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. + unit: rps + queries: + - expr: sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: request-memory + - expr: sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: query-memory + - title: Datanode Write Failures + type: timeseries + description: Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. + unit: eps + queries: + - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: region-request-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: failed-insert-[{{instance}}]-[{{pod}}] + - title: Buffered Ingestion Loss + type: timeseries + description: Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. + unit: eps + queries: + - expr: sum(rate(greptime_pending_rows_flush_failures[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-failures + - expr: sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: dropped-rows + - title: Mito Backpressure and Failures + type: timeseries + description: Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. + unit: eps + queries: + - expr: sum(rate(greptime_mito_write_reject_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: write-reject + - expr: sum(rate(greptime_mito_write_stall_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: write-stall + - expr: sum(rate(greptime_mito_flush_failure_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-failure + - expr: sum(rate(greptime_mito_compaction_failure_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-failure + - title: Scan and Compaction Memory Rejects + type: timeseries + description: Datanode scan and compaction memory rejection/exhaustion counters. + unit: rps + queries: + - expr: sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: scan-rejected + - expr: sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: scan-exhausted + - expr: sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-rejected + - title: OpenDAL Errors + type: timeseries + description: Object-store errors by scheme, operation, and error, excluding NotFound noise. + unit: eps + queries: + - expr: sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{scheme}}-{{operation}}-{{error}}' + - title: Metasrv Failures + type: timeseries + description: Region migration and reconciliation failures in metasrv. + unit: eps + queries: + - expr: sum(rate(greptime_meta_region_migration_fail[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: migration-fail + - expr: sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: reconciliation-error + - title: Flow and Trigger Failures + type: timeseries + description: Derived-data and alerting pipeline failures. + unit: eps + queries: + - expr: sum by (code) (rate(greptime_flow_errors[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flow-{{code}} + - expr: sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trigger-evaluate + - expr: sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trigger-send-alert + - expr: sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: trigger-save-alert + - title: Mito GC Failures + type: timeseries + description: Mito garbage-collection errors and skipped/orphaned files on datanodes. + unit: short + queries: + - expr: sum(rate(greptime_mito_gc_errors_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: gc-errors + - expr: sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: orphaned-index-files + - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: skipped-unparsable + - title: Capacity + panels: + - title: Runtime Threads + type: timeseries + description: Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_runtime_threads_alive) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: alive-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_runtime_threads_idle) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: idle-[{{instance}}]-[{{pod}}] + - title: Request Memory Utilization + type: timeseries + description: Frontend request memory usage divided by configured request memory limit. + unit: percentunit + queries: + - expr: sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Query Memory Usage + type: timeseries + description: Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. unit: bytes queries: - - expr: sum(process_resident_memory_bytes{}) by (instance, pod) + - expr: sum by (instance, pod) (greptime_query_memory_pool_usage_bytes) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{ pod }}]' - - expr: max(greptime_memory_limit_in_bytes{}) + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Scan and Compaction Memory + type: timeseries + description: Datanode scan memory usage and compaction memory utilization. + unit: bytes + queries: + - expr: sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: scan-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-used-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-limit-[{{instance}}]-[{{pod}}] + - title: Write Buffer and Active Stalling + type: timeseries + description: Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. + unit: bytes + queries: + - expr: sum by (instance, pod) (greptime_mito_write_buffer_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: buffer-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_write_stalling_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: stalling-[{{instance}}]-[{{pod}}] + - title: Prom Store Backlog + type: timeseries + description: Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_prom_store_pending_rows) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: rows-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_prom_store_pending_batches) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: batches-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_prom_store_pending_workers) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: workers-[{{instance}}]-[{{pod}}] + - title: Inflight Flush and Compaction + type: timeseries + description: Current in-flight flush and compaction tasks on datanodes. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_mito_inflight_flush_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_inflight_compaction_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-[{{instance}}]-[{{pod}}] + - title: Resources + panels: + - title: Frontend CPU Usage per Instance + type: timeseries + description: Current cpu usage by instance + unit: none + queries: + - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu' + - expr: max(greptime_cpu_limit_in_millicores) datasource: type: prometheus uid: ${metrics} @@ -197,57 +684,12 @@ groups: description: Current cpu usage by instance unit: none queries: - - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_cpu_limit_in_millicores{}) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: limit - - title: Frontend Memory per Instance - type: timeseries - description: Current memory usage by instance - unit: bytes - queries: - - expr: sum(process_resident_memory_bytes{}) by (instance, pod) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_memory_limit_in_bytes{}) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: limit - - title: Frontend CPU Usage per Instance - type: timeseries - description: Current cpu usage by instance - unit: none - queries: - - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu' - - expr: max(greptime_cpu_limit_in_millicores{}) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: limit - - title: Metasrv Memory per Instance - type: timeseries - description: Current memory usage by instance - unit: bytes - queries: - - expr: sum(process_resident_memory_bytes{}) by (instance, pod) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{ pod }}]-resident' - - expr: max(greptime_memory_limit_in_bytes{}) + - expr: max(greptime_cpu_limit_in_millicores) datasource: type: prometheus uid: ${metrics} @@ -257,27 +699,57 @@ groups: description: Current cpu usage by instance unit: none queries: - - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_cpu_limit_in_millicores{}) + - expr: max(greptime_cpu_limit_in_millicores) datasource: type: prometheus uid: ${metrics} legendFormat: limit - - title: Flownode Memory per Instance + - title: Frontend Memory per Instance type: timeseries description: Current memory usage by instance unit: bytes queries: - - expr: sum(process_resident_memory_bytes{}) by (instance, pod) + - expr: sum(process_resident_memory_bytes) by (instance, pod) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_memory_limit_in_bytes{}) + - expr: max(greptime_memory_limit_in_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: limit + - title: Datanode Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: bytes + queries: + - expr: sum(process_resident_memory_bytes) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{ pod }}]' + - expr: max(greptime_memory_limit_in_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: limit + - title: Metasrv Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: bytes + queries: + - expr: sum(process_resident_memory_bytes) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]-resident' + - expr: max(greptime_memory_limit_in_bytes) datasource: type: prometheus uid: ${metrics} @@ -287,16 +759,193 @@ groups: description: Current cpu usage by instance unit: none queries: - - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod) + - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]' - - expr: max(greptime_cpu_limit_in_millicores{}) + - expr: max(greptime_cpu_limit_in_millicores) datasource: type: prometheus uid: ${metrics} legendFormat: limit + - title: Flownode Memory per Instance + type: timeseries + description: Current memory usage by instance + unit: bytes + queries: + - expr: sum(process_resident_memory_bytes) by (instance, pod) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{ pod }}]' + - expr: max(greptime_memory_limit_in_bytes) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: limit + - title: Queries + panels: + - title: Query Rate by Protocol + type: timeseries + description: Query API call rates by protocol, collected from frontends. + unit: reqps + queries: + - expr: sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql + - expr: sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres + - expr: sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql + - expr: sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: http-sql + - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-query + - title: Query Latency by Protocol + type: timeseries + description: p95, p99, and average query latency by main frontend protocol. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-p95 + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-p99 + - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-p95 + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-p99 + - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql-p95 + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql-p99 + - expr: sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-avg + - expr: sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-avg + - expr: sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: promql-avg + - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: grpc-query-avg + - title: Query Stage Latency + type: timeseries + description: p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p95-{{stage}} + - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99-{{stage}} + - title: Merge Scan Fan-out and Errors + type: timeseries + description: Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_query_merge_scan_regions) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: regions-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: errors-[{{instance}}]-[{{pod}}] + - title: Pushdown Fallback Errors + type: timeseries + description: Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. + unit: eps + queries: + - expr: sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: pushdown-fallback-errors + - title: PromQL Series Count + type: timeseries + description: Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_promql_series_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Connections and Prepared Statements + type: timeseries + description: MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. + unit: short + queries: + - expr: sum by (instance, pod) (greptime_servers_mysql_connection_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-connections-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_servers_postgres_connection_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-connections-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: mysql-prepared-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: postgres-prepared-[{{instance}}]-[{{pod}}] - title: Frontend Requests panels: - title: HTTP QPS per Instance @@ -309,9 +958,9 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]' - - title: HTTP P99 per Instance + - title: HTTP P99 and Avg per Instance type: timeseries - description: HTTP P99 per Instance. + description: HTTP P99 and average per Instance. unit: s queries: - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health|/metrics"}[$__rate_interval]))) @@ -319,99 +968,114 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - expr: sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~"/health|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg' - title: gRPC QPS per Instance type: timeseries description: gRPC QPS per Instance. unit: reqps queries: - - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval])) + - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]' - - title: gRPC P99 per Instance + - title: gRPC P99 and Avg per Instance type: timeseries - description: gRPC P99 per Instance. + description: gRPC P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99' + - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg' - title: MySQL QPS per Instance type: timeseries description: MySQL QPS per Instance. unit: reqps queries: - - expr: sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval])) + - expr: sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: MySQL P99 per Instance + - title: MySQL P99 and Avg per Instance type: timeseries - description: MySQL P99 per Instance. + description: MySQL P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{ instance }}]-[{{ pod }}]-p99' + - expr: sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-avg' - title: PostgreSQL QPS per Instance type: timeseries description: PostgreSQL QPS per Instance. unit: reqps queries: - - expr: sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval])) + - expr: sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: PostgreSQL P99 per Instance + - title: PostgreSQL P99 and Avg per Instance type: timeseries - description: PostgreSQL P99 per Instance. + description: PostgreSQL P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - title: Frontend to Datanode - panels: - - title: Ingest Rows per Instance - type: timeseries - description: Ingestion rate by row as in each frontend - unit: rowsps - queries: - - expr: sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval])) + - expr: sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' + legendFormat: '[{{instance}}]-[{{pod}}]-avg' + - title: Frontend to Datanode + panels: - title: Region Call QPS per Instance type: timeseries description: Region Call QPS per Instance. unit: ops queries: - - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval])) + - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' - - title: Region Call P99 per Instance + - title: Region Call P99 and Avg per Instance type: timeseries - description: Region Call P99 per Instance. + description: Region Call P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]' - - title: 'Frontend Handle Bulk Insert Elapsed Time ' + - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg' + - title: Frontend Handle Bulk Insert Elapsed Time type: timeseries description: Per-stage time for frontend to handle bulk insert requests unit: s @@ -426,34 +1090,156 @@ groups: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95' - - title: Mito Engine + - title: Datanode + panels: + - title: Region Request Failures and Failed Inserts + type: timeseries + description: Datanode region request failures and failed inserts by instance. + unit: eps + queries: + - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: request-fail-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: failed-insert-[{{instance}}]-[{{pod}}] + - title: Write Rejects and Stalls + type: timeseries + description: Mito write rejects, write stall events, and active write stalling by datanode. + unit: short + queries: + - expr: sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: reject-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: stall-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (greptime_mito_write_stalling_count) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: active-stalling-[{{instance}}]-[{{pod}}] + - title: Flush and Compaction Failures + type: timeseries + description: Mito flush and compaction failure rates by datanode. + unit: eps + queries: + - expr: sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: flush-[{{instance}}]-[{{pod}}] + - expr: sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: compaction-[{{instance}}]-[{{pod}}] + - title: Mito GC Health + type: timeseries + description: Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. + unit: short + queries: + - expr: sum(rate(greptime_mito_gc_runs_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: runs + - expr: sum(rate(greptime_mito_gc_errors_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: errors + - expr: sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: files-deleted + - expr: sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: orphaned-index-files + - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: skipped-unparsable + - title: Mito GC Duration + type: timeseries + description: P99 and average Mito garbage-collection duration by stage. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{stage}}-p99' + - expr: sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{stage}}-avg' + - title: Storage panels: - title: Request OPS per Instance type: timeseries description: Request QPS per Instance. unit: ops queries: - - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval])) + - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' - - title: Request P99 per Instance + - title: Request P99 and Avg per Instance type: timeseries - description: Request P99 per Instance. + description: Request P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' + - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg' + - title: Request Wait P99 and Avg per Worker + type: timeseries + description: Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p95' + - expr: histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p99' + - expr: sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-avg' - title: Write Buffer per Instance type: timeseries description: Write Buffer per Instance. unit: decbytes queries: - - expr: greptime_mito_write_buffer_bytes{} + - expr: greptime_mito_write_buffer_bytes datasource: type: prometheus uid: ${metrics} @@ -463,26 +1249,7 @@ groups: description: Ingestion size by row counts. unit: rowsps queries: - - expr: sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Flush OPS per Instance - type: timeseries - description: Flush QPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]' - - title: Write Stall per Instance - type: timeseries - description: Write Stall per Instance. - queries: - - expr: sum by(instance, pod) (greptime_mito_write_stall_total{}) + - expr: sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval])) datasource: type: prometheus uid: ${metrics} @@ -492,151 +1259,51 @@ groups: description: Read Stage OPS per Instance. unit: ops queries: - - expr: sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval])) + - expr: sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage="total"}[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Read Stage P99 per Instance + - title: Read Stage P99 and Avg per Instance type: timeseries - description: Read Stage P99 per Instance. + description: Read Stage P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' - - title: Write Stage P99 per Instance - type: timeseries - description: Write Stage P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' - - title: Compaction OPS per Instance - type: timeseries - description: Compaction OPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{ instance }}]-[{{pod}}]' - - title: Compaction Elapsed Time per Instance by Stage - type: timeseries - description: Compaction latency by stage - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99' - - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval])) + - expr: sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg' - - title: Compaction P99 per Instance + - title: Write Stage P99 and Avg per Instance type: timeseries - description: Compaction P99 per Instance. + description: Write Stage P99 and average per Instance. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval]))) + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction' - - title: WAL write size - type: timeseries - description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. - unit: bytes - queries: - - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]' + - expr: sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95' - - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99' - - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-throughput' + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg' - title: Cached Bytes per Instance type: timeseries description: Cached Bytes per Instance. unit: decbytes queries: - - expr: greptime_mito_cache_bytes{} + - expr: greptime_mito_cache_bytes datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' - - title: Inflight Compaction - type: timeseries - description: Ongoing compaction task count - unit: none - queries: - - expr: greptime_mito_inflight_compaction_count - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: WAL sync duration seconds - type: timeseries - description: Raft engine (local disk) log store sync latency, p99 - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - title: Log Store op duration seconds - type: timeseries - description: Write-ahead log operations latency at p99 - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99' - - title: Inflight Flush - type: timeseries - description: Ongoing flush task count - unit: none - queries: - - expr: greptime_mito_inflight_flush_count - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Compaction Input/Output Bytes - type: timeseries - description: Compaction oinput output bytes - unit: bytes - queries: - - expr: sum by(instance, pod) (greptime_mito_compaction_input_bytes) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-input' - - expr: sum by(instance, pod) (greptime_mito_compaction_output_bytes) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-output' - title: Region Worker Handle Bulk Insert Requests type: timeseries description: Per-stage elapsed time for region worker to handle bulk insert region requests. @@ -654,7 +1321,7 @@ groups: legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG' - title: Active Series and Field Builders Count type: timeseries - description: Compaction oinput output bytes + description: Active series and field-builder counts per memtable by instance. unit: none queries: - expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count) @@ -686,166 +1353,287 @@ groups: type: timeseries description: The local cache miss of the datanode. queries: - - expr: sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval])) + - expr: sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval])) datasource: type: prometheus uid: ${metrics} legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]' - - title: OpenDAL + - title: Flush and Compaction panels: - - title: QPS per Instance + - title: Flush OPS per Instance type: timeseries - description: QPS per Instance. + description: Flush QPS per Instance. unit: ops queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval])) + - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Read QPS per Instance + legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]' + - title: Flush Elapsed Time type: timeseries - description: Read QPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read|Reader::read"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Read P99 per Instance - type: timeseries - description: Read P99 per Instance. + description: Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. unit: s queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read|Reader::read"}[$__rate_interval]))) + - expr: histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Write QPS per Instance + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' + - expr: histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - expr: sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg' + - title: Flush Throughput type: timeseries - description: Write QPS per Instance. - unit: ops + description: Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. + unit: Bps queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) + - expr: sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Write P99 per Instance - type: timeseries - description: Write P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write|Writer::close|write"}[$__rate_interval]))) + legendFormat: '[{{instance}}]-[{{pod}}]-bytes' + - expr: sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: List QPS per Instance + legendFormat: '[{{instance}}]-[{{pod}}]-files' + - title: Inflight Flush type: timeseries - description: List QPS per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' - - title: List P99 per Instance - type: timeseries - description: List P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' - - title: Other Requests per Instance - type: timeseries - description: Other Requests per Instance. - unit: ops - queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|write|list|stat"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Other Request P99 per Instance - type: timeseries - description: Other Request P99 per Instance. - unit: s - queries: - - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read|write|list|Writer::write|Writer::close|Reader::read"}[$__rate_interval]))) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: Opendal traffic - type: timeseries - description: Total traffic as in bytes by instance and operation - unit: decbytes - queries: - - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' - - title: OpenDAL errors per Instance - type: timeseries - description: OpenDAL error counts per Instance. - queries: - - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]' - - title: Remote WAL - panels: - - title: Triggered region flush total - type: timeseries - description: Triggered region flush total + description: Ongoing flush task count unit: none queries: - - expr: meta_triggered_region_flush_total + - expr: greptime_mito_inflight_flush_count datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{topic_name}}' - - title: Triggered region checkpoint total + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Compaction OPS per Instance type: timeseries - description: Triggered region checkpoint total + description: Compaction OPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{ instance }}]-[{{pod}}]' + - title: Inflight Compaction + type: timeseries + description: Ongoing compaction task count unit: none queries: - - expr: meta_triggered_region_checkpoint_total + - expr: greptime_mito_inflight_compaction_count datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{topic_name}}' - - title: Topic estimated replay size + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Compaction P99 and Avg per Instance type: timeseries - description: Topic estimated max replay size + description: Compaction P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-avg' + - title: Compaction Elapsed Time per Instance by Stage + type: timeseries + description: Compaction latency by stage + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99' + - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg' + - title: Compaction Input/Output Bytes + type: timeseries + description: Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. + unit: Bps + queries: + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-input' + - expr: sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-output' + - title: Index + panels: + - title: Index Apply Elapsed Time + type: timeseries + description: Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-p95' + - expr: histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-p99' + - title: Index Create Elapsed Time + type: timeseries + description: Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. + unit: s + queries: + - expr: histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-{{stage}}-p95' + - expr: histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-{{stage}}-p99' + - title: Index Create Rows and Bytes + type: timeseries + description: Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. + unit: rowsps + queries: + - expr: sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-rows' + - expr: sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-bytes' + - title: Index Memory Usage + type: timeseries + description: Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. unit: bytes queries: - - expr: meta_topic_estimated_replay_size + - expr: greptime_index_apply_memory_usage datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{topic_name}}' - - title: Kafka logstore's bytes traffic + legendFormat: apply + - expr: sum by (type) (greptime_index_create_memory_usage) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: create-{{type}} + - title: Index IO Bytes type: timeseries - description: Kafka logstore's bytes traffic - unit: bytes + description: Index read and write byte rates by operation and file type for puffin and intermediate files. + unit: Bps queries: - - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval]) + - expr: sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval])) datasource: type: prometheus uid: ${metrics} - legendFormat: '{{pod}}-{{logstore}}' + legendFormat: '{{type}}-{{file_type}}' + - title: Index IO Operations + type: timeseries + description: Index IO operation rates by operation and file type, including read, write, seek, and flush operations. + unit: ops + queries: + - expr: sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{type}}-{{file_type}}' + - title: Index Cache + type: timeseries + description: Index-related cache hits, misses, and evictions from Mito caches. + unit: ops + queries: + - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|vector_index|index_result"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: hit-{{type}} + - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|vector_index|index_result"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: miss-{{type}} + - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|vector_index|index_result"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: evict-{{type}}-{{cause}} - title: Metasrv panels: + - title: Inactive and Lease-expired Regions + type: timeseries + description: Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. + unit: short + queries: + - expr: sum(greptime_meta_inactive_regions) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: inactive-regions + - expr: sum(greptime_lease_expired_region) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: lease-expired + - title: Heartbeat Health + type: timeseries + description: Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. + unit: short + queries: + - expr: sum(rate(greptime_meta_heartbeat_rate[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: meta-recv-rate + - expr: sum(greptime_meta_heartbeat_connection_num) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: meta-connections + - expr: sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: frontend-send + - expr: sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: frontend-recv + - expr: sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-send + - expr: sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: datanode-recv - title: Region migration datanode type: status-history description: Counter of region migration by source and destination @@ -865,7 +1653,7 @@ groups: description: Counter of region migration error unit: none queries: - - expr: greptime_meta_region_migration_error + - expr: rate(greptime_meta_region_migration_error[$__rate_interval]) datasource: type: prometheus uid: ${metrics} @@ -900,6 +1688,11 @@ groups: type: prometheus uid: ${metrics} legendFormat: '{{pod}} {{op}} {{type}} {{result}} p90' + - expr: sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg' - title: Handler Execution Latency type: timeseries description: | @@ -914,20 +1707,25 @@ groups: type: prometheus uid: ${metrics} legendFormat: '{{pod}} {{name}} p90' + - expr: sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{name}}]-avg' - title: Heartbeat Packet Size type: timeseries description: | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload. unit: bytes queries: - - expr: histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket)) + - expr: histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '{{pod}}' - title: Meta Heartbeat Receive Rate type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: Rate of heartbeats received by metasrv from datanodes and frontends. unit: s queries: - expr: rate(greptime_meta_heartbeat_rate[$__rate_interval]) @@ -937,17 +1735,22 @@ groups: legendFormat: '{{pod}}' - title: Meta KV Ops Latency type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: p99 and average latency of metasrv key-value store operations by op and target. unit: s queries: - - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket)) + - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '{{pod}}-{{op}} p99' + - expr: sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{op}}]-[{{target}}]-avg' - title: Rate of meta KV Ops type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: Rate of metasrv key-value store operations by op. unit: none queries: - expr: rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]) @@ -957,35 +1760,35 @@ groups: legendFormat: '{{pod}}-{{op}} p99' - title: DDL Latency type: timeseries - description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. + description: p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. unit: s queries: - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateLogicalTables-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateTable-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateView-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: CreateFlow-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: DropTable-{{step}} p90 - - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table)) + - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} @@ -993,9 +1796,9 @@ groups: - title: Reconciliation stats type: timeseries description: Reconciliation stats - unit: s + unit: ops queries: - - expr: greptime_meta_reconciliation_stats + - expr: rate(greptime_meta_reconciliation_stats[$__rate_interval]) datasource: type: prometheus uid: ${metrics} @@ -1005,154 +1808,11 @@ groups: description: Elapsed of Reconciliation steps unit: s queries: - - expr: histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket) + - expr: histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval]))) datasource: type: prometheus uid: ${metrics} legendFormat: '{{procedure_name}}-{{step}}-P90' - - title: Flownode - panels: - - title: Flow Ingest / Output Rate - type: timeseries - description: Flow Ingest / Output Rate. - queries: - - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]' - - title: Flow Ingest Latency - type: timeseries - description: Flow Ingest Latency. - queries: - - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p95' - - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - title: Flow Operation Latency - type: timeseries - description: Flow Operation Latency. - queries: - - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' - - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' - - title: Flow Buffer Size per Instance - type: timeseries - description: Flow Buffer Size per Instance. - queries: - - expr: greptime_flow_input_buf_size - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]' - - title: Flow Processing Error per Instance - type: timeseries - description: Flow Processing Error per Instance. - queries: - - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval])) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]' - - title: Trigger - panels: - - title: Trigger Count - type: timeseries - description: Total number of triggers currently defined. - queries: - - expr: greptime_trigger_count{} - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - - title: Trigger Eval Elapsed - type: timeseries - description: Elapsed time for trigger evaluation, including query execution and condition evaluation. - unit: s - queries: - - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p99' - - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-p75' - - title: Trigger Eval Failure Rate - type: timeseries - description: Rate of failed trigger evaluations. - unit: none - queries: - - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval]) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - - title: Send Alert Elapsed - type: timeseries - description: Elapsed time to send trigger alerts to notification channels. - unit: s - queries: - - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99' - - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75' - - title: Send Alert Failure Rate - type: timeseries - description: Rate of failures when sending trigger alerts. - unit: none - queries: - - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval]) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - - title: Save Alert Elapsed - type: timeseries - description: Elapsed time to persist trigger alert records. - unit: s - queries: - - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99' - - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)" - datasource: - type: prometheus - uid: ${metrics} - legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75' - - title: Save Alert Failure Rate - type: timeseries - description: Rate of failures when persisting trigger alert records. - unit: none - queries: - - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]) - datasource: - type: prometheus - uid: ${metrics} - legendFormat: __auto - title: Hotspot panels: - title: Hotspot Regions @@ -1310,3 +1970,379 @@ groups: type: prometheus uid: ${metrics} legendFormat: '{{stage}}' + - title: Object Store + panels: + - title: QPS per Instance + type: timeseries + description: QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Read QPS per Instance + type: timeseries + description: Read QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read|Reader::read"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Read P99 and Avg per Instance + type: timeseries + description: Read P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read|Reader::read"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"read|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read|Reader::read"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: Write QPS per Instance + type: timeseries + description: Write QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Write P99 and Avg per Instance + type: timeseries + description: Write P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ "Writer::write|Writer::close|write"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: List QPS per Instance + type: timeseries + description: List QPS per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - title: List P99 and Avg per Instance + type: timeseries + description: List P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="list"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: Other Requests per Instance + type: timeseries + description: Other Requests per Instance. + unit: ops + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: Other Request P99 and Avg per Instance + type: timeseries + description: Other Request P99 and average per Instance. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg' + - title: Opendal traffic + type: timeseries + description: Total traffic as in bytes by instance and operation + unit: decbytes + queries: + - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]' + - title: OpenDAL errors per Instance + type: timeseries + description: OpenDAL error counts per Instance. + queries: + - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]' + - title: WAL + panels: + - title: WAL write size + type: timeseries + description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. + unit: bytes + queries: + - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95' + - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99' + - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-throughput' + - title: WAL sync duration seconds + type: timeseries + description: Raft engine (local disk) log store sync latency, p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - title: Log Store op duration seconds + type: timeseries + description: Write-ahead log operations latency at p99 + unit: s + queries: + - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99' + - title: Triggered region flush total + type: timeseries + description: Triggered region flush total + unit: none + queries: + - expr: meta_triggered_region_flush_total + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{topic_name}}' + - title: Triggered region checkpoint total + type: timeseries + description: Triggered region checkpoint total + unit: none + queries: + - expr: meta_triggered_region_checkpoint_total + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{topic_name}}' + - title: Topic estimated replay size + type: timeseries + description: Topic estimated max replay size + unit: bytes + queries: + - expr: meta_topic_estimated_replay_size + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{topic_name}}' + - title: Kafka logstore's bytes traffic + type: timeseries + description: Kafka logstore's bytes traffic + unit: bytes + queries: + - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '{{pod}}-{{logstore}}' + - title: Flownode + panels: + - title: Flow Ingest / Output Rate + type: timeseries + description: Flow Ingest / Output Rate. + queries: + - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]' + - title: Flow Ingest Latency + type: timeseries + description: Flow Ingest Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-p99' + - expr: sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-avg' + - title: Flow Operation Latency + type: timeseries + description: Flow Operation Latency. + queries: + - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95' + - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type)) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99' + - title: Flow Buffer Size per Instance + type: timeseries + description: Flow Buffer Size per Instance. + queries: + - expr: greptime_flow_input_buf_size + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]' + - title: Flow Processing Error per Instance + type: timeseries + description: Flow Processing Error per Instance. + queries: + - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]' + - title: Trigger + panels: + - title: Trigger Count + type: timeseries + description: Total number of triggers currently defined. + queries: + - expr: greptime_trigger_count + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Trigger Eval Elapsed + type: timeseries + description: Elapsed time for trigger evaluation, including query execution and condition evaluation. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99 + - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p75 + - expr: sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: avg + - title: Trigger Eval Failure Rate + type: timeseries + description: Rate of failed trigger evaluations. + unit: none + queries: + - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Send Alert Elapsed + type: timeseries + description: Elapsed time to send trigger alerts to notification channels. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99 + - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p75 + - expr: sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: avg + - title: Send Alert Failure Rate + type: timeseries + description: Rate of failures when sending trigger alerts. + unit: none + queries: + - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto + - title: Save Alert Elapsed + type: timeseries + description: Elapsed time to persist trigger alert records. + unit: s + queries: + - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p99 + - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval]))) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: p75 + - expr: sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval])) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: avg + - title: Save Alert Failure Rate + type: timeseries + description: Rate of failures when persisting trigger alert records. + unit: none + queries: + - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]) + datasource: + type: prometheus + uid: ${metrics} + legendFormat: __auto diff --git a/grafana/scripts/gen-dashboards.sh b/grafana/scripts/gen-dashboards.sh index 9cf8cbd2a3a..5087d5bde9c 100755 --- a/grafana/scripts/gen-dashboards.sh +++ b/grafana/scripts/gen-dashboards.sh @@ -6,7 +6,11 @@ DAC_IMAGE=ghcr.io/zyy17/dac:20250423-522bd35 remove_instance_filters() { # Remove the instance filters for the standalone dashboards. - sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\",?//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" > "$STANDALONE_DASHBOARD_DIR/dashboard.json" + sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\"[[:space:]]*,?[[:space:]]*//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" \ + | sed -E 's/\{[[:space:]]*,[[:space:]]*/{/g' \ + | sed -E 's/,[[:space:]]*\}/}/g' \ + | sed -E 's/([A-Za-z_:][A-Za-z0-9_:]*)\{[[:space:]]*\}/\1/g' \ + > "$STANDALONE_DASHBOARD_DIR/dashboard.json" } generate_intermediate_dashboards_and_docs() {