diff --git a/grafana/dashboards/metrics/cluster/dashboard.json b/grafana/dashboards/metrics/cluster/dashboard.json
index 7792dbedaf..35a0dced91 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.json
+++ b/grafana/dashboards/metrics/cluster/dashboard.json
@@ -25,7 +25,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
- "id": 5,
+ "id": 13,
"links": [],
"panels": [
{
@@ -59,8 +59,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -93,7 +92,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"datasource": {
@@ -131,8 +130,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -167,7 +165,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -220,8 +218,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -253,7 +250,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"datasource": {
@@ -287,8 +284,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -320,7 +316,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -371,8 +367,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -404,7 +399,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -455,8 +450,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -491,7 +485,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -632,8 +626,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
}
@@ -664,7 +657,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -805,8 +798,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -838,7 +830,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -1783,7 +1775,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 50
+ "y": 138
},
"id": 269,
"options": {
@@ -1883,7 +1875,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 50
+ "y": 138
},
"id": 271,
"options": {
@@ -1986,7 +1978,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 60
+ "y": 148
},
"id": 272,
"options": {
@@ -2086,7 +2078,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 60
+ "y": 148
},
"id": 273,
"options": {
@@ -2160,6 +2152,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2190,8 +2183,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2200,7 +2192,8 @@
]
},
"unit": "reqps"
- }
+ },
+ "overrides": []
},
"gridPos": {
"h": 8,
@@ -2260,6 +2253,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2290,8 +2284,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2361,6 +2354,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2391,8 +2385,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2462,6 +2455,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2492,8 +2486,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2561,6 +2554,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2591,8 +2585,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2662,6 +2655,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2692,8 +2686,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2764,6 +2757,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2794,8 +2788,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2865,6 +2858,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2895,8 +2889,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -3402,7 +3395,7 @@
"type": "row"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@@ -3410,2220 +3403,2354 @@
"y": 46
},
"id": 293,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
+ "panels": [],
+ "title": "Mito Engine",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "description": "Request QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 127
- },
- "id": 294,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Request P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 127
- },
- "id": 295,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Buffer per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 135
- },
- "id": 296,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Buffer per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ingestion size by row counts.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 135
- },
- "id": 297,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Rows per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Flush QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 143
- },
- "id": 298,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Flush OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stall per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 143
- },
- "id": 299,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stall per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 151
- },
- "id": 300,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 151
- },
- "id": 301,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 159
- },
- "id": 302,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 159
- },
- "id": 303,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction latency by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 167
- },
- "id": 304,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Elapsed Time per Instance by Stage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 167
- },
- "id": 305,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 175
- },
- "id": 306,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "WAL write size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Cached Bytes per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 175
- },
- "id": 307,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cached Bytes per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing compaction task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 183
- },
- "id": 308,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_compaction_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Compaction",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Raft engine (local disk) log store sync latency, p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 183
- },
- "id": 310,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "WAL sync duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead log operations latency at p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 191
- },
- "id": 311,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Log Store op duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing flush task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 191
- },
- "id": 312,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_flush_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Flush",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 199
- },
- "id": 335,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-input",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-output",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Input/Output Bytes",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
{
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[bulk_extend]-AVG",
- "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
- "[minipc-1:4000]-[]-[write_bulk]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
}
]
},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 199
- },
- "id": 337,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "id": 294,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
],
- "title": "Region Worker Handle Bulk Insert Requests",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "id": 295,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Buffer per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 55
+ },
+ "id": 296,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Buffer per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ingestion size by row counts.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 55
+ },
+ "id": 297,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Rows per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Flush QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 63
+ },
+ "id": 298,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Flush OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stall per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 63
+ },
+ "id": 299,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stall per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 71
+ },
+ "id": 300,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 71
+ },
+ "id": 301,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 79
+ },
+ "id": 302,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 79
+ },
+ "id": 303,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction latency by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 87
+ },
+ "id": 304,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
+ "range": true,
+ "refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Per-stage elapsed time for region worker to decode requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 207
- },
- "id": 338,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region Worker Convert Requests",
- "type": "timeseries"
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Mito Engine",
- "type": "row"
+ "title": "Compaction Elapsed Time per Instance by Stage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 87
+ },
+ "id": 305,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 95
+ },
+ "id": 306,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "WAL write size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Cached Bytes per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 95
+ },
+ "id": 307,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cached Bytes per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing compaction task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 103
+ },
+ "id": 308,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_compaction_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Compaction",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Raft engine (local disk) log store sync latency, p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 103
+ },
+ "id": 310,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "WAL sync duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead log operations latency at p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 111
+ },
+ "id": 311,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Log Store op duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing flush task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 111
+ },
+ "id": 312,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_flush_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Flush",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 119
+ },
+ "id": 335,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-input",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-output",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Input/Output Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[bulk_extend]-AVG",
+ "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
+ "[minipc-1:4000]-[]-[write_bulk]-AVG"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 119
+ },
+ "id": 337,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Handle Bulk Insert Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 127
+ },
+ "id": 348,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-series",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Active Series and Field Builders Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to decode requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 127
+ },
+ "id": 338,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Convert Requests",
+ "type": "timeseries"
},
{
"collapsed": true,
@@ -5631,7 +5758,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 47
+ "y": 135
},
"id": 313,
"panels": [
@@ -5700,7 +5827,7 @@
"h": 10,
"w": 24,
"x": 0,
- "y": 1228
+ "y": 1316
},
"id": 314,
"options": {
@@ -5801,7 +5928,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1238
+ "y": 1326
},
"id": 315,
"options": {
@@ -5902,7 +6029,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1238
+ "y": 1326
},
"id": 316,
"options": {
@@ -6003,7 +6130,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1245
+ "y": 1333
},
"id": 317,
"options": {
@@ -6104,7 +6231,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1245
+ "y": 1333
},
"id": 318,
"options": {
@@ -6205,7 +6332,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1252
+ "y": 1340
},
"id": 319,
"options": {
@@ -6307,7 +6434,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1252
+ "y": 1340
},
"id": 320,
"options": {
@@ -6409,7 +6536,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1259
+ "y": 1347
},
"id": 321,
"options": {
@@ -6510,7 +6637,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1259
+ "y": 1347
},
"id": 322,
"options": {
@@ -6611,7 +6738,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1266
+ "y": 1354
},
"id": 323,
"options": {
@@ -6715,7 +6842,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1266
+ "y": 1354
},
"id": 334,
"options": {
@@ -6756,1197 +6883,1219 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 48
+ "y": 136
},
"id": 324,
- "panels": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration by source and destination",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "fillOpacity": 70,
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineWidth": 1
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 137
+ },
+ "id": 325,
+ "options": {
+ "colWidth": 0.9,
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "rowHeight": 0.9,
+ "showValue": "auto",
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
+ "instant": false,
+ "legendFormat": "from-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "to-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region migration datanode",
+ "type": "status-history"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration error",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 137
+ },
+ "id": 326,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_error",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region migration error",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 145
+ },
+ "id": 327,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_datanode_load",
+ "instant": false,
+ "legendFormat": "Datanode-{{datanode_id}}-writeload",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Datanode load",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 145
+ },
+ "id": 339,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rate of SQL Executions (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Measures the response time of SQL executions via the RDS backend. ",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 153
+ },
+ "id": 340,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "SQL Execution Latency (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 153
+ },
+ "id": 341,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{name}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Handler Execution Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 161
+ },
+ "id": 342,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Heartbeat Packet Size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 161
+ },
+ "id": 345,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta Heartbeat Receive Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 169
+ },
+ "id": 343,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta KV Ops Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 169
+ },
+ "id": 346,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Rate of meta KV Ops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 177
+ },
+ "id": 347,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateLogicalTables-{{step}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateTable-{{step}} p90",
+ "range": true,
+ "refId": "B",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateView-{{step}} p90",
+ "range": true,
+ "refId": "C",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateFlow-{{step}} p90",
+ "range": true,
+ "refId": "D",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "DropTable-{{step}} p90",
+ "range": true,
+ "refId": "E",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "AlterTable-{{step}} p90",
+ "range": true,
+ "refId": "F",
+ "useBackend": false
+ }
+ ],
+ "title": "DDL Latency",
+ "type": "timeseries"
+ }
+ ],
"title": "Metasrv",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration by source and destination",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "fillOpacity": 70,
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineWidth": 1
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 49
- },
- "id": 325,
- "options": {
- "colWidth": 0.9,
- "legend": {
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "rowHeight": 0.9,
- "showValue": "auto",
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
- "instant": false,
- "legendFormat": "from-datanode-{{datanode_id}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
- "hide": false,
- "instant": false,
- "legendFormat": "to-datanode-{{datanode_id}}",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region migration datanode",
- "type": "status-history"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration error",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 49
- },
- "id": 326,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_error",
- "instant": false,
- "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region migration error",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 57
- },
- "id": 327,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_datanode_load",
- "instant": false,
- "legendFormat": "Datanode-{{datanode_id}}-writeload",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Datanode load",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 57
- },
- "id": 339,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Rate of SQL Executions (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Measures the response time of SQL executions via the RDS backend. ",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 65
- },
- "id": 340,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "SQL Execution Latency (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 65
- },
- "id": 341,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{name}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Handler Execution Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 73
- },
- "id": 342,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Heartbeat Packet Size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 73
- },
- "id": 345,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta Heartbeat Receive Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 81
- },
- "id": 343,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta KV Ops Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 81
- },
- "id": 346,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Rate of meta KV Ops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 89
- },
- "id": 347,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateLogicalTables-{{step}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateTable-{{step}} p90",
- "range": true,
- "refId": "B",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateView-{{step}} p90",
- "range": true,
- "refId": "C",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateFlow-{{step}} p90",
- "range": true,
- "refId": "D",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "DropTable-{{step}} p90",
- "range": true,
- "refId": "E",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "AlterTable-{{step}} p90",
- "range": true,
- "refId": "F",
- "useBackend": false
- }
- ],
- "title": "DDL Latency",
- "type": "timeseries"
- },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 97
+ "y": 137
},
"id": 328,
"panels": [
@@ -8013,7 +8162,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 2372
+ "y": 2460
},
"id": 329,
"options": {
@@ -8108,7 +8257,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 2372
+ "y": 2460
},
"id": 330,
"options": {
@@ -8216,7 +8365,7 @@
"h": 8,
"w": 9,
"x": 0,
- "y": 2380
+ "y": 2468
},
"id": 331,
"options": {
@@ -8324,7 +8473,7 @@
"h": 8,
"w": 9,
"x": 9,
- "y": 2380
+ "y": 2468
},
"id": 332,
"options": {
@@ -8419,7 +8568,7 @@
"h": 8,
"w": 6,
"x": 18,
- "y": 2380
+ "y": 2468
},
"id": 333,
"options": {
@@ -8456,48 +8605,40 @@
"type": "row"
}
],
+ "preload": false,
"refresh": "10s",
- "schemaVersion": 39,
+ "schemaVersion": 41,
"tags": [],
"templating": {
"list": [
{
"current": {
- "selected": false,
- "text": "metrics",
- "value": "P177A7EA3611FE6B1"
+ "text": "prometheus",
+ "value": "cemcvbpaqhq0wc"
},
- "hide": 0,
"includeAll": false,
- "multi": false,
"name": "metrics",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
- "skipUrlSync": false,
"type": "datasource"
},
{
"current": {
- "selected": false,
- "text": "information_schema",
- "value": "P0CE5E4D2C4819379"
+ "text": "",
+ "value": ""
},
- "hide": 0,
"includeAll": false,
- "multi": false,
"name": "information_schema",
"options": [],
"query": "mysql",
"refresh": 1,
"regex": "",
- "skipUrlSync": false,
"type": "datasource"
},
{
"current": {
- "selected": false,
"text": "All",
"value": "$__all"
},
diff --git a/grafana/dashboards/metrics/cluster/dashboard.md b/grafana/dashboards/metrics/cluster/dashboard.md
index 4abbaabcb9..0a7c941621 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.md
+++ b/grafana/dashboards/metrics/cluster/dashboard.md
@@ -70,6 +70,7 @@
| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`
`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`
`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
+| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`
`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
# OpenDAL
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
diff --git a/grafana/dashboards/metrics/cluster/dashboard.yaml b/grafana/dashboards/metrics/cluster/dashboard.yaml
index 6ee30840e7..4314f45625 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.yaml
+++ b/grafana/dashboards/metrics/cluster/dashboard.yaml
@@ -612,6 +612,21 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG'
+ - title: Active Series and Field Builders Count
+ type: timeseries
+ description: Compaction oinput output bytes
+ unit: none
+ queries:
+ - expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-series'
+ - expr: sum by(instance, pod) (greptime_mito_memtable_field_builder_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-field_builders'
- title: Region Worker Convert Requests
type: timeseries
description: Per-stage elapsed time for region worker to decode requests.
diff --git a/grafana/dashboards/metrics/standalone/dashboard.json b/grafana/dashboards/metrics/standalone/dashboard.json
index f3826a3c0a..bd2d1893bd 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.json
+++ b/grafana/dashboards/metrics/standalone/dashboard.json
@@ -25,7 +25,7 @@
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
- "id": 5,
+ "id": 13,
"links": [],
"panels": [
{
@@ -59,8 +59,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -93,7 +92,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"datasource": {
@@ -131,8 +130,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -167,7 +165,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -220,8 +218,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -253,7 +250,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"datasource": {
@@ -287,8 +284,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -320,7 +316,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -371,8 +367,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -404,7 +399,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -455,8 +450,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -491,7 +485,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -632,8 +626,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
}
@@ -664,7 +657,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -805,8 +798,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
}
]
},
@@ -838,7 +830,7 @@
"textMode": "auto",
"wideLayout": true
},
- "pluginVersion": "11.1.3",
+ "pluginVersion": "12.0.0",
"targets": [
{
"dataset": "information_schema",
@@ -1783,7 +1775,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 50
+ "y": 138
},
"id": 269,
"options": {
@@ -1883,7 +1875,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 50
+ "y": 138
},
"id": 271,
"options": {
@@ -1986,7 +1978,7 @@
"h": 10,
"w": 12,
"x": 0,
- "y": 60
+ "y": 148
},
"id": 272,
"options": {
@@ -2086,7 +2078,7 @@
"h": 10,
"w": 12,
"x": 12,
- "y": 60
+ "y": 148
},
"id": 273,
"options": {
@@ -2160,6 +2152,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2190,8 +2183,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2200,7 +2192,8 @@
]
},
"unit": "reqps"
- }
+ },
+ "overrides": []
},
"gridPos": {
"h": 8,
@@ -2260,6 +2253,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2290,8 +2284,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2361,6 +2354,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2391,8 +2385,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2462,6 +2455,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2492,8 +2486,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2561,6 +2554,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2591,8 +2585,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2662,6 +2655,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2692,8 +2686,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2764,6 +2757,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2794,8 +2788,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -2865,6 +2858,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "points",
"fillOpacity": 0,
"gradientMode": "none",
@@ -2895,8 +2889,7 @@
"mode": "absolute",
"steps": [
{
- "color": "green",
- "value": null
+ "color": "green"
},
{
"color": "red",
@@ -3402,7 +3395,7 @@
"type": "row"
},
{
- "collapsed": true,
+ "collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
@@ -3410,2220 +3403,2354 @@
"y": 46
},
"id": 293,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
+ "panels": [],
+ "title": "Mito Engine",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- "description": "Request QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
},
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 127
- },
- "id": 294,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
},
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
}
},
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Request P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 127
- },
- "id": 295,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Request P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Buffer per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 135
- },
- "id": 296,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_write_buffer_bytes{}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Buffer per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ingestion size by row counts.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 135
- },
- "id": 297,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Rows per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Flush QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 143
- },
- "id": 298,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Flush OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stall per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 143
- },
- "id": 299,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stall per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 151
- },
- "id": 300,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 151
- },
- "id": 301,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 159
- },
- "id": 302,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 159
- },
- "id": 303,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction latency by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 167
- },
- "id": 304,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Elapsed Time per Instance by Stage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 167
- },
- "id": 305,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 175
- },
- "id": 306,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "WAL write size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Cached Bytes per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 175
- },
- "id": 307,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_cache_bytes{}",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Cached Bytes per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing compaction task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 183
- },
- "id": 308,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_compaction_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Compaction",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Raft engine (local disk) log store sync latency, p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 183
- },
- "id": 310,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "WAL sync duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead log operations latency at p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 191
- },
- "id": 311,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Log Store op duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing flush task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 191
- },
- "id": 312,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_flush_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Flush",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 199
- },
- "id": 335,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-input",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-output",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Input/Output Bytes",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
{
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[bulk_extend]-AVG",
- "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
- "[minipc-1:4000]-[]-[write_bulk]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
}
]
},
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 199
- },
- "id": 337,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 47
+ },
+ "id": 294,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
],
- "title": "Region Worker Handle Bulk Insert Requests",
- "type": "timeseries"
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 47
+ },
+ "id": 295,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Buffer per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 55
+ },
+ "id": 296,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_write_buffer_bytes{}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Buffer per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ingestion size by row counts.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 55
+ },
+ "id": 297,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Rows per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Flush QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 63
+ },
+ "id": 298,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Flush OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stall per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 63
+ },
+ "id": 299,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stall per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 71
+ },
+ "id": 300,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 71
+ },
+ "id": 301,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stage P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 79
+ },
+ "id": 302,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write Stage P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 79
+ },
+ "id": 303,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction latency by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 87
+ },
+ "id": 304,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
+ "range": true,
+ "refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Per-stage elapsed time for region worker to decode requests.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 207
- },
- "id": 338,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region Worker Convert Requests",
- "type": "timeseries"
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Mito Engine",
- "type": "row"
+ "title": "Compaction Elapsed Time per Instance by Stage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction P99 per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 87
+ },
+ "id": 305,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Compaction P99 per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 95
+ },
+ "id": 306,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "WAL write size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Cached Bytes per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 95
+ },
+ "id": 307,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_cache_bytes{}",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Cached Bytes per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing compaction task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 103
+ },
+ "id": 308,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_compaction_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Compaction",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Raft engine (local disk) log store sync latency, p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 103
+ },
+ "id": 310,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "WAL sync duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead log operations latency at p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 111
+ },
+ "id": 311,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Log Store op duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing flush task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 111
+ },
+ "id": 312,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_flush_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Flush",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 119
+ },
+ "id": 335,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-input",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-output",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Input/Output Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": [
+ {
+ "__systemRef": "hideSeriesFrom",
+ "matcher": {
+ "id": "byNames",
+ "options": {
+ "mode": "exclude",
+ "names": [
+ "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[bulk_extend]-AVG",
+ "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
+ "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
+ "[minipc-1:4000]-[]-[write_bulk]-AVG"
+ ],
+ "prefix": "All except:",
+ "readOnly": true
+ }
+ },
+ "properties": [
+ {
+ "id": "custom.hideFrom",
+ "value": {
+ "legend": false,
+ "tooltip": false,
+ "viz": true
+ }
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 119
+ },
+ "id": 337,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Handle Bulk Insert Requests",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction oinput output bytes",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 127
+ },
+ "id": 348,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-series",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Active Series and Field Builders Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Per-stage elapsed time for region worker to decode requests.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 127
+ },
+ "id": 338,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Worker Convert Requests",
+ "type": "timeseries"
},
{
"collapsed": true,
@@ -5631,7 +5758,7 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 47
+ "y": 135
},
"id": 313,
"panels": [
@@ -5700,7 +5827,7 @@
"h": 10,
"w": 24,
"x": 0,
- "y": 1228
+ "y": 1316
},
"id": 314,
"options": {
@@ -5801,7 +5928,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1238
+ "y": 1326
},
"id": 315,
"options": {
@@ -5902,7 +6029,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1238
+ "y": 1326
},
"id": 316,
"options": {
@@ -6003,7 +6130,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1245
+ "y": 1333
},
"id": 317,
"options": {
@@ -6104,7 +6231,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1245
+ "y": 1333
},
"id": 318,
"options": {
@@ -6205,7 +6332,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1252
+ "y": 1340
},
"id": 319,
"options": {
@@ -6307,7 +6434,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1252
+ "y": 1340
},
"id": 320,
"options": {
@@ -6409,7 +6536,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1259
+ "y": 1347
},
"id": 321,
"options": {
@@ -6510,7 +6637,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1259
+ "y": 1347
},
"id": 322,
"options": {
@@ -6611,7 +6738,7 @@
"h": 7,
"w": 12,
"x": 0,
- "y": 1266
+ "y": 1354
},
"id": 323,
"options": {
@@ -6715,7 +6842,7 @@
"h": 7,
"w": 12,
"x": 12,
- "y": 1266
+ "y": 1354
},
"id": 334,
"options": {
@@ -6756,1197 +6883,1219 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 48
+ "y": 136
},
"id": 324,
- "panels": [],
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration by source and destination",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisPlacement": "auto",
+ "fillOpacity": 70,
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "lineWidth": 1
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 137
+ },
+ "id": 325,
+ "options": {
+ "colWidth": 0.9,
+ "legend": {
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": false
+ },
+ "rowHeight": 0.9,
+ "showValue": "auto",
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
+ "instant": false,
+ "legendFormat": "from-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "to-datanode-{{datanode_id}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region migration datanode",
+ "type": "status-history"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Counter of region migration error",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 137
+ },
+ "id": 326,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_meta_region_migration_error",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region migration error",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 145
+ },
+ "id": 327,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_datanode_load",
+ "instant": false,
+ "legendFormat": "Datanode-{{datanode_id}}-writeload",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Datanode load",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 145
+ },
+ "id": 339,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Rate of SQL Executions (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Measures the response time of SQL executions via the RDS backend. ",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ms"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 153
+ },
+ "id": 340,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "SQL Execution Latency (RDS)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 153
+ },
+ "id": 341,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}} {{name}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Handler Execution Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 161
+ },
+ "id": 342,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Heartbeat Packet Size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 161
+ },
+ "id": 345,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta Heartbeat Receive Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 169
+ },
+ "id": 343,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Meta KV Ops Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 169
+ },
+ "id": 346,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "{{pod}}-{{op}} p99",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Rate of meta KV Ops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 177
+ },
+ "id": 347,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateLogicalTables-{{step}} p90",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateTable-{{step}} p90",
+ "range": true,
+ "refId": "B",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateView-{{step}} p90",
+ "range": true,
+ "refId": "C",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "CreateFlow-{{step}} p90",
+ "range": true,
+ "refId": "D",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "DropTable-{{step}} p90",
+ "range": true,
+ "refId": "E",
+ "useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
+ "fullMetaSearch": false,
+ "hide": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "AlterTable-{{step}} p90",
+ "range": true,
+ "refId": "F",
+ "useBackend": false
+ }
+ ],
+ "title": "DDL Latency",
+ "type": "timeseries"
+ }
+ ],
"title": "Metasrv",
"type": "row"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration by source and destination",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "fillOpacity": 70,
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "lineWidth": 1
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 49
- },
- "id": 325,
- "options": {
- "colWidth": 0.9,
- "legend": {
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": false
- },
- "rowHeight": 0.9,
- "showValue": "auto",
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.1.3",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}",
- "instant": false,
- "legendFormat": "from-datanode-{{datanode_id}}",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}",
- "hide": false,
- "instant": false,
- "legendFormat": "to-datanode-{{datanode_id}}",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Region migration datanode",
- "type": "status-history"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Counter of region migration error",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 49
- },
- "id": 326,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_meta_region_migration_error",
- "instant": false,
- "legendFormat": "{{pod}}-{{state}}-{{error_type}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region migration error",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 57
- },
- "id": 327,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_datanode_load",
- "instant": false,
- "legendFormat": "Datanode-{{datanode_id}}-writeload",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Datanode load",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 57
- },
- "id": 339,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])",
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Rate of SQL Executions (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Measures the response time of SQL executions via the RDS backend. ",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ms"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 65
- },
- "id": 340,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "SQL Execution Latency (RDS)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 65
- },
- "id": 341,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}} {{name}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Handler Execution Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 73
- },
- "id": 342,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Heartbeat Packet Size",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 73
- },
- "id": 345,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta Heartbeat Receive Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 81
- },
- "id": 343,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Meta KV Ops Latency",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 81
- },
- "id": 346,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "{{pod}}-{{op}} p99",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Rate of meta KV Ops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green",
- "value": null
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 89
- },
- "id": 347,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "mode": "single",
- "sort": "none"
- }
- },
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateLogicalTables-{{step}} p90",
- "range": true,
- "refId": "A",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateTable-{{step}} p90",
- "range": true,
- "refId": "B",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateView-{{step}} p90",
- "range": true,
- "refId": "C",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "CreateFlow-{{step}} p90",
- "range": true,
- "refId": "D",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "DropTable-{{step}} p90",
- "range": true,
- "refId": "E",
- "useBackend": false
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
- "fullMetaSearch": false,
- "hide": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "AlterTable-{{step}} p90",
- "range": true,
- "refId": "F",
- "useBackend": false
- }
- ],
- "title": "DDL Latency",
- "type": "timeseries"
- },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 97
+ "y": 137
},
"id": 328,
"panels": [
@@ -8013,7 +8162,7 @@
"h": 8,
"w": 12,
"x": 0,
- "y": 2372
+ "y": 2460
},
"id": 329,
"options": {
@@ -8108,7 +8257,7 @@
"h": 8,
"w": 12,
"x": 12,
- "y": 2372
+ "y": 2460
},
"id": 330,
"options": {
@@ -8216,7 +8365,7 @@
"h": 8,
"w": 9,
"x": 0,
- "y": 2380
+ "y": 2468
},
"id": 331,
"options": {
@@ -8324,7 +8473,7 @@
"h": 8,
"w": 9,
"x": 9,
- "y": 2380
+ "y": 2468
},
"id": 332,
"options": {
@@ -8419,7 +8568,7 @@
"h": 8,
"w": 6,
"x": 18,
- "y": 2380
+ "y": 2468
},
"id": 333,
"options": {
@@ -8456,48 +8605,40 @@
"type": "row"
}
],
+ "preload": false,
"refresh": "10s",
- "schemaVersion": 39,
+ "schemaVersion": 41,
"tags": [],
"templating": {
"list": [
{
"current": {
- "selected": false,
- "text": "metrics",
- "value": "P177A7EA3611FE6B1"
+ "text": "prometheus",
+ "value": "cemcvbpaqhq0wc"
},
- "hide": 0,
"includeAll": false,
- "multi": false,
"name": "metrics",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
- "skipUrlSync": false,
"type": "datasource"
},
{
"current": {
- "selected": false,
- "text": "information_schema",
- "value": "P0CE5E4D2C4819379"
+ "text": "",
+ "value": ""
},
- "hide": 0,
"includeAll": false,
- "multi": false,
"name": "information_schema",
"options": [],
"query": "mysql",
"refresh": 1,
"regex": "",
- "skipUrlSync": false,
"type": "datasource"
},
{
"current": {
- "selected": false,
"text": "All",
"value": "$__all"
},
diff --git a/grafana/dashboards/metrics/standalone/dashboard.md b/grafana/dashboards/metrics/standalone/dashboard.md
index 072312d7d3..049424a061 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.md
+++ b/grafana/dashboards/metrics/standalone/dashboard.md
@@ -70,6 +70,7 @@
| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`
`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`
`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
+| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`
`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
# OpenDAL
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
diff --git a/grafana/dashboards/metrics/standalone/dashboard.yaml b/grafana/dashboards/metrics/standalone/dashboard.yaml
index 9505de7b4b..d6706d65d1 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.yaml
+++ b/grafana/dashboards/metrics/standalone/dashboard.yaml
@@ -612,6 +612,21 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG'
+ - title: Active Series and Field Builders Count
+ type: timeseries
+ description: Compaction oinput output bytes
+ unit: none
+ queries:
+ - expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-series'
+ - expr: sum by(instance, pod) (greptime_mito_memtable_field_builder_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-field_builders'
- title: Region Worker Convert Requests
type: timeseries
description: Per-stage elapsed time for region worker to decode requests.
diff --git a/src/mito2/src/flush.rs b/src/mito2/src/flush.rs
index 3ec7ef4f72..b84aaedc27 100644
--- a/src/mito2/src/flush.rs
+++ b/src/mito2/src/flush.rs
@@ -340,13 +340,16 @@ impl RegionFlushTask {
let memtables = version.memtables.immutables();
let mut file_metas = Vec::with_capacity(memtables.len());
let mut flushed_bytes = 0;
+ let mut series_count = 0;
for mem in memtables {
if mem.is_empty() {
// Skip empty memtables.
continue;
}
- let max_sequence = mem.stats().max_sequence();
+ let stats = mem.stats();
+ let max_sequence = stats.max_sequence();
+ series_count += stats.series_count();
let iter = mem.iter(None, None, None)?;
let source = Source::Iter(iter);
@@ -396,10 +399,11 @@ impl RegionFlushTask {
let file_ids: Vec<_> = file_metas.iter().map(|f| f.file_id).collect();
info!(
- "Successfully flush memtables, region: {}, reason: {}, files: {:?}, cost: {:?}s",
+ "Successfully flush memtables, region: {}, reason: {}, files: {:?}, series count: {}, cost: {:?}s",
self.region_id,
self.reason.as_str(),
file_ids,
+ series_count,
timer.stop_and_record(),
);
diff --git a/src/mito2/src/memtable.rs b/src/mito2/src/memtable.rs
index b810beaaf8..25ad15da55 100644
--- a/src/mito2/src/memtable.rs
+++ b/src/mito2/src/memtable.rs
@@ -86,6 +86,8 @@ pub struct MemtableStats {
num_ranges: usize,
/// The maximum sequence number in the memtable.
max_sequence: SequenceNumber,
+ /// Number of estimated timeseries in memtable.
+ series_count: usize,
}
impl MemtableStats {
@@ -120,6 +122,11 @@ impl MemtableStats {
pub fn max_sequence(&self) -> SequenceNumber {
self.max_sequence
}
+
+ /// Series count in memtable.
+ pub fn series_count(&self) -> usize {
+ self.series_count
+ }
}
pub type BoxedBatchIterator = Box> + Send>;
diff --git a/src/mito2/src/memtable/partition_tree.rs b/src/mito2/src/memtable/partition_tree.rs
index 83c1efe301..8307f84980 100644
--- a/src/mito2/src/memtable/partition_tree.rs
+++ b/src/mito2/src/memtable/partition_tree.rs
@@ -227,6 +227,7 @@ impl Memtable for PartitionTreeMemtable {
num_rows: 0,
num_ranges: 0,
max_sequence: 0,
+ series_count: 0,
};
}
@@ -241,12 +242,14 @@ impl Memtable for PartitionTreeMemtable {
.expect("Timestamp column must have timestamp type");
let max_timestamp = ts_type.create_timestamp(self.max_timestamp.load(Ordering::Relaxed));
let min_timestamp = ts_type.create_timestamp(self.min_timestamp.load(Ordering::Relaxed));
+ let series_count = self.tree.series_count();
MemtableStats {
estimated_bytes,
time_range: Some((min_timestamp, max_timestamp)),
num_rows: self.num_rows.load(Ordering::Relaxed),
num_ranges: 1,
max_sequence: self.max_sequence.load(Ordering::Relaxed),
+ series_count,
}
}
diff --git a/src/mito2/src/memtable/partition_tree/partition.rs b/src/mito2/src/memtable/partition_tree/partition.rs
index 804396771e..6e455bbfca 100644
--- a/src/mito2/src/memtable/partition_tree/partition.rs
+++ b/src/mito2/src/memtable/partition_tree/partition.rs
@@ -308,6 +308,10 @@ impl Partition {
.map(|meta| meta.column_schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME)
.unwrap_or(false)
}
+
+ pub(crate) fn series_count(&self) -> usize {
+ self.inner.read().unwrap().series_count()
+ }
}
pub(crate) struct PartitionStats {
@@ -577,4 +581,9 @@ impl Inner {
}
Ok(())
}
+
+ /// Returns count of timeseries.
+ fn series_count(&self) -> usize {
+ self.pk_to_pk_id.len()
+ }
}
diff --git a/src/mito2/src/memtable/partition_tree/tree.rs b/src/mito2/src/memtable/partition_tree/tree.rs
index 58224566fc..909714ca92 100644
--- a/src/mito2/src/memtable/partition_tree/tree.rs
+++ b/src/mito2/src/memtable/partition_tree/tree.rs
@@ -439,6 +439,16 @@ impl PartitionTree {
metrics.partitions_after_pruning = pruned.len();
pruned
}
+
+ /// Returns all series count in all partitions.
+ pub(crate) fn series_count(&self) -> usize {
+ self.partitions
+ .read()
+ .unwrap()
+ .values()
+ .map(|p| p.series_count())
+ .sum()
+ }
}
#[derive(Default)]
diff --git a/src/mito2/src/memtable/simple_bulk_memtable.rs b/src/mito2/src/memtable/simple_bulk_memtable.rs
index ad97bd7077..eca99f0d54 100644
--- a/src/mito2/src/memtable/simple_bulk_memtable.rs
+++ b/src/mito2/src/memtable/simple_bulk_memtable.rs
@@ -33,6 +33,7 @@ use crate::memtable::{
AllocTracker, BoxedBatchIterator, IterBuilder, KeyValues, Memtable, MemtableId, MemtableRange,
MemtableRangeContext, MemtableRanges, MemtableRef, MemtableStats,
};
+use crate::metrics::MEMTABLE_ACTIVE_SERIES_COUNT;
use crate::read::dedup::LastNonNullIter;
use crate::read::scan_region::PredicateGroup;
use crate::read::Batch;
@@ -52,6 +53,12 @@ pub struct SimpleBulkMemtable {
series: RwLock,
}
+impl Drop for SimpleBulkMemtable {
+ fn drop(&mut self) {
+ MEMTABLE_ACTIVE_SERIES_COUNT.dec();
+ }
+}
+
impl SimpleBulkMemtable {
pub(crate) fn new(
id: MemtableId,
@@ -278,6 +285,7 @@ impl Memtable for SimpleBulkMemtable {
num_rows: 0,
num_ranges: 0,
max_sequence: 0,
+ series_count: 0,
};
}
let ts_type = self
@@ -296,6 +304,7 @@ impl Memtable for SimpleBulkMemtable {
num_rows,
num_ranges: 1,
max_sequence: self.max_sequence.load(Ordering::Relaxed),
+ series_count: 1,
}
}
diff --git a/src/mito2/src/memtable/time_series.rs b/src/mito2/src/memtable/time_series.rs
index 63ab8cebed..6757b0926c 100644
--- a/src/mito2/src/memtable/time_series.rs
+++ b/src/mito2/src/memtable/time_series.rs
@@ -55,7 +55,10 @@ use crate::memtable::{
MemtableId, MemtableRange, MemtableRangeContext, MemtableRanges, MemtableRef, MemtableStats,
PredicateGroup,
};
-use crate::metrics::{READ_ROWS_TOTAL, READ_STAGE_ELAPSED};
+use crate::metrics::{
+ MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT, MEMTABLE_ACTIVE_SERIES_COUNT, READ_ROWS_TOTAL,
+ READ_STAGE_ELAPSED,
+};
use crate::read::dedup::LastNonNullIter;
use crate::read::{Batch, BatchBuilder, BatchColumn};
use crate::region::options::MergeMode;
@@ -316,7 +319,7 @@ impl Memtable for TimeSeriesMemtable {
}
fn is_empty(&self) -> bool {
- self.series_set.series.read().unwrap().is_empty()
+ self.series_set.series.read().unwrap().0.is_empty()
}
fn freeze(&self) -> Result<()> {
@@ -336,6 +339,7 @@ impl Memtable for TimeSeriesMemtable {
num_rows: 0,
num_ranges: 0,
max_sequence: 0,
+ series_count: 0,
};
}
let ts_type = self
@@ -348,12 +352,14 @@ impl Memtable for TimeSeriesMemtable {
.expect("Timestamp column must have timestamp type");
let max_timestamp = ts_type.create_timestamp(self.max_timestamp.load(Ordering::Relaxed));
let min_timestamp = ts_type.create_timestamp(self.min_timestamp.load(Ordering::Relaxed));
+ let series_count = self.series_set.series.read().unwrap().0.len();
MemtableStats {
estimated_bytes,
time_range: Some((min_timestamp, max_timestamp)),
num_rows: self.num_rows.load(Ordering::Relaxed),
num_ranges: 1,
max_sequence: self.max_sequence.load(Ordering::Relaxed),
+ series_count,
}
}
@@ -368,13 +374,27 @@ impl Memtable for TimeSeriesMemtable {
}
}
-type SeriesRwLockMap = RwLock, Arc>>>;
+#[derive(Default)]
+struct SeriesMap(BTreeMap, Arc>>);
+
+impl Drop for SeriesMap {
+ fn drop(&mut self) {
+ let num_series = self.0.len();
+ let num_field_builders = self
+ .0
+ .values()
+ .map(|v| v.read().unwrap().active.num_field_builders())
+ .sum::();
+ MEMTABLE_ACTIVE_SERIES_COUNT.sub(num_series as i64);
+ MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT.sub(num_field_builders as i64);
+ }
+}
#[derive(Clone)]
pub(crate) struct SeriesSet {
- pub(crate) region_metadata: RegionMetadataRef,
- pub(crate) series: Arc,
- pub(crate) codec: Arc,
+ region_metadata: RegionMetadataRef,
+ series: Arc>,
+ codec: Arc,
}
impl SeriesSet {
@@ -390,7 +410,7 @@ impl SeriesSet {
impl SeriesSet {
/// Push [KeyValue] to SeriesSet with given primary key and return key/value allocated memory size.
fn push_to_series(&self, primary_key: Vec, kv: &KeyValue) -> (usize, usize) {
- if let Some(series) = self.series.read().unwrap().get(&primary_key) {
+ if let Some(series) = self.series.read().unwrap().0.get(&primary_key) {
let value_allocated = series.write().unwrap().push(
kv.timestamp(),
kv.sequence(),
@@ -401,7 +421,7 @@ impl SeriesSet {
};
let mut indices = self.series.write().unwrap();
- match indices.entry(primary_key) {
+ match indices.0.entry(primary_key) {
Entry::Vacant(v) => {
let key_len = v.key().len();
let mut series = Series::new(&self.region_metadata);
@@ -425,7 +445,7 @@ impl SeriesSet {
#[cfg(test)]
fn get_series(&self, primary_key: &[u8]) -> Option>> {
- self.series.read().unwrap().get(primary_key).cloned()
+ self.series.read().unwrap().0.get(primary_key).cloned()
}
/// Iterates all series in [SeriesSet].
@@ -492,7 +512,7 @@ struct Metrics {
struct Iter {
metadata: RegionMetadataRef,
- series: Arc,
+ series: Arc>,
projection: HashSet,
last_key: Option>,
predicate: Vec,
@@ -508,7 +528,7 @@ impl Iter {
#[allow(clippy::too_many_arguments)]
pub(crate) fn try_new(
metadata: RegionMetadataRef,
- series: Arc,
+ series: Arc>,
projection: HashSet,
predicate: Option,
pk_schema: arrow::datatypes::SchemaRef,
@@ -565,10 +585,10 @@ impl Iterator for Iter {
let start = Instant::now();
let map = self.series.read().unwrap();
let range = match &self.last_key {
- None => map.range::, _>(..),
- Some(last_key) => {
- map.range::, _>((Bound::Excluded(last_key), Bound::Unbounded))
- }
+ None => map.0.range::, _>(..),
+ Some(last_key) => map
+ .0
+ .range::, _>((Bound::Excluded(last_key), Bound::Unbounded)),
};
// TODO(hl): maybe yield more than one time series to amortize range overhead.
@@ -668,6 +688,7 @@ pub(crate) struct Series {
impl Series {
pub(crate) fn with_capacity(region_metadata: &RegionMetadataRef, builder_cap: usize) -> Self {
+ MEMTABLE_ACTIVE_SERIES_COUNT.inc();
Self {
pk_cache: None,
active: ValueBuilder::new(region_metadata, builder_cap),
@@ -788,7 +809,6 @@ impl ValueBuilder {
.map(|c| c.column_schema.data_type.clone())
.collect::>();
let fields = (0..field_types.len()).map(|_| None).collect();
-
Self {
timestamp: Vec::with_capacity(capacity),
timestamp_type,
@@ -799,6 +819,11 @@ impl ValueBuilder {
}
}
+ /// Returns number of field builders.
+ pub fn num_field_builders(&self) -> usize {
+ self.fields.iter().flatten().count()
+ }
+
/// Pushes a new row to `ValueBuilder`.
/// We don't need primary keys since they've already be encoded.
/// Returns the size of field values.
@@ -842,6 +867,7 @@ impl ValueBuilder {
mutable_vector.push_nulls(num_rows - 1);
let _ = mutable_vector.push(field_value);
self.fields[idx] = Some(mutable_vector);
+ MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT.inc();
}
}
}
@@ -1028,6 +1054,7 @@ impl From for Values {
.enumerate()
.map(|(i, v)| {
if let Some(v) = v {
+ MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT.dec();
v.finish()
} else {
let mut single_null = value.field_types[i].create_mutable_vector(num_rows);
diff --git a/src/mito2/src/metrics.rs b/src/mito2/src/metrics.rs
index 107f94b515..c9ebbc0fbf 100644
--- a/src/mito2/src/metrics.rs
+++ b/src/mito2/src/metrics.rs
@@ -414,6 +414,18 @@ lazy_static! {
"greptime_mito_compaction_output_bytes",
"mito compaction output file size",
).unwrap();
+
+ /// Active series count in TimeSeriesMemtable
+ pub static ref MEMTABLE_ACTIVE_SERIES_COUNT: IntGauge = register_int_gauge!(
+ "greptime_mito_memtable_active_series_count",
+ "active time series count in TimeSeriesMemtable",
+ ).unwrap();
+
+ /// Active field builder count in TimeSeriesMemtable
+ pub static ref MEMTABLE_ACTIVE_FIELD_BUILDER_COUNT: IntGauge = register_int_gauge!(
+ "greptime_mito_memtable_field_builder_count",
+ "active field builder count in TimeSeriesMemtable",
+ ).unwrap();
}
/// Stager notifier to collect metrics.