diff --git a/grafana/README.md b/grafana/README.md index 3167cee95a..233dcdd4d6 100644 --- a/grafana/README.md +++ b/grafana/README.md @@ -5,6 +5,13 @@ GreptimeDB's official Grafana dashboard. Status notify: we are still working on this config. It's expected to change frequently in the recent days. Please feel free to submit your feedback and/or contribution to this dashboard 🤗 +If you use Helm [chart](https://github.com/GreptimeTeam/helm-charts) to deploy GreptimeDB cluster, you can enable self-monitoring by setting the following values in your Helm chart: + +- `monitoring.enabled=true`: Deploys a standalone GreptimeDB instance dedicated to monitoring the cluster; +- `grafana.enabled=true`: Deploys Grafana and automatically imports the monitoring dashboard; + +The standalone GreptimeDB instance will collect metrics from your cluster and the dashboard will be available in the Grafana UI. For detailed deployment instructions, please refer to our [Kubernetes deployment guide](https://docs.greptime.com/nightly/user-guide/deployments/deploy-on-kubernetes/getting-started). + # How to use ## `greptimedb.json` diff --git a/grafana/greptimedb-cluster.json b/grafana/greptimedb-cluster.json index f101d1f181..72d5a862df 100644 --- a/grafana/greptimedb-cluster.json +++ b/grafana/greptimedb-cluster.json @@ -1,35 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS-1", - "label": "prometheus-1", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "10.1.5" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], "annotations": { "list": [ { @@ -52,11 +21,10 @@ } ] }, - "description": "", + "description": "The Grafana dashboard of GreptimeDB cluster.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, "links": [], "liveNow": false, "panels": [ @@ -68,28 +36,1312 @@ "x": 0, "y": 0 }, - "id": 155, + "id": 189, "panels": [], - "title": "Frontend Entry Middleware", + "title": "Overview", "type": "row" }, { "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "type": "mysql", + "uid": "${information_schema}" }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 239, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/^pkg_version$/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT pkg_version FROM information_schema.build_info", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Version", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 2, + "y": 1 + }, + "id": 240, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_resident{pod=~\"$frontend\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Frontend Memory", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 5, + "y": 1 + }, + "id": 241, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Frontend Num", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 7, + "y": 1 + }, + "id": 242, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_resident{pod=~\"$datanode\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Datanode Memory", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 10, + "y": 1 + }, + "id": 243, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Datanode Num", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 244, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_resident{pod=~\"$metasrv\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Metasrv Memory", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 15, + "y": 1 + }, + "id": 245, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT count(*) FROM information_schema.cluster_info WHERE peer_type = 'METASRV';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Metasrv Num", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 4 + }, + "id": 246, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Databases", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 4 + }, + "id": 247, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) FROM information_schema.tables WHERE table_schema != 'information_schema'", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Tables", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 6, + "y": 4 + }, + "id": 248, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Storage Size", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "max": 2, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 10, + "y": 4 + }, + "id": 249, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Ingest Rows Rate", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 8 + }, + "id": 250, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(memtable_size) * 0.42825 from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "WAL Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 8 + }, + "id": 251, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(index_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Index Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 8 + }, + "id": 252, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(manifest_size) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Manifest Size", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 6, + "y": 8 + }, + "id": 253, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(region_id) FROM information_schema.region_peers", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Regions Count", + "type": "stat" + }, + { + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "sishort" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 8, + "y": 8 + }, + "id": 254, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.3", + "targets": [ + { + "dataset": "information_schema", + "datasource": { + "type": "mysql", + "uid": "${information_schema}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Region Rows", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "points", + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -127,57 +1379,26 @@ "value": 80 } ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] } - ] + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 6, + "w": 24, "x": 0, - "y": 1 + "y": 11 }, - "id": 152, + "id": 193, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -185,60 +1406,64 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "sum (rate(greptime_servers_http_prometheus_write_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{instance}}-{{path}}-{{code}}-p99", + "legendFormat": "prometheus-remote-write", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, "editorMode": "code", - "exemplar": false, - "expr": "sum by(instance, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": true, - "includeNullMetadata": true, + "expr": "sum (rate(greptime_servers_grpc_requests_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", + "hide": false, "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{path}}-{{code}}-qps", + "legendFormat": "gRPC", "range": true, - "refId": "B", - "useBackend": false + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_http_logs_ingestion_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "http_logs", + "range": true, + "refId": "C" } ], - "title": "gRPC middleware", + "title": "Ingestion", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "points", + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -276,2040 +1501,26 @@ "value": 80 } ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] } - ] + }, + "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 154, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~\"$instance\",path!~\"/health|/metrics\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{path}}-{{method}}-{{code}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(instance, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$instance\",path!~\"/health|/metrics\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{path}}-{{method}}-{{code}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP middleware", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 156, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, subprotocol, db) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{subprotocol}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(instance, subprotocol, db) (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{subprotocol}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "MySQL per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 157, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, subprotocol, db) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{subprotocol}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(instance, subprotocol, db) (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{subprotocol}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "PostgreSQL per DB", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, + "h": 6, "w": 24, "x": 0, "y": 17 }, - "id": 158, - "panels": [], - "title": "Frontend HTTP per DB", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 18 - }, - "id": 159, + "id": 255, "options": { "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_sql_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_sql_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP sql per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 18 - }, - "id": 160, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_promql_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP promql per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 18 - }, - "id": 161, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_influxdb_write_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_influxdb_write_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP influxdb per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 26 - }, - "id": 162, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP prom store write per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 26 - }, - "id": 183, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_prometheus_read_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_prometheus_read_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP prom store read per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 26 - }, - "id": 184, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP otlp metrics per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 26 - }, - "id": 185, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "HTTP otlp traces per DB", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 - }, - "id": 163, - "panels": [], - "title": "Frontend gRPC per DB", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 35 - }, - "id": 164, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "label_replace(histogram_quantile(0.99, sum by(instance, le, db, type, code) (rate(greptime_servers_grpc_db_request_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval]))), \"db\", \"$1\", \"db\", \"(.*)-public\")", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-{{type}}-{{code}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db, type, code) (rate(greptime_servers_grpc_db_request_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-{{type}}-{{code}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "gRPC per DB", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 35 - }, - "id": 165, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": false - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, db) (rate(greptime_servers_grpc_prom_request_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{db}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, db) (rate(greptime_servers_grpc_prom_request_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{db}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "gRPC prom per DB", - "type": "timeseries" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, - "id": 166, - "panels": [], - "title": "Frontend-Datanode Call", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-rps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "rowsps" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 44 - }, - "id": 186, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true, - "sortBy": "Name", - "sortDesc": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "rate(greptime_table_operator_ingest_rows{instance=~\"$instance\"}[$__rate_interval])", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-rps", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "Ingested rows", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 44 - }, - "id": 167, - "options": { - "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2317,41 +1528,43 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{instance}}-{{request_type}}-p99", + "legendFormat": "mysql", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, request_type) (rate(greptime_grpc_region_request_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", "hide": false, - "includeNullMetadata": true, "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{request_type}}-qps", + "legendFormat": "pg", "range": true, - "refId": "B", - "useBackend": false + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{pod=~\"$frontend\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "promql", + "range": true, + "refId": "C" } ], - "title": "gRPC region call", + "title": "Queries", "type": "timeseries" }, { @@ -2360,24 +1573,26 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 23 }, - "id": 168, + "id": 237, "panels": [], - "title": "Datanode Mito", + "title": "Resources", "type": "row" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, + "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2413,157 +1628,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "points" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 53 - }, - "id": 188, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{type}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{type}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Handle request", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2576,18 +1641,22 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 53 + "h": 10, + "w": 8, + "x": 0, + "y": 24 }, - "id": 187, + "id": 234, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Last *", + "sortDesc": false }, "tooltip": { "mode": "single", @@ -2598,27 +1667,49 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "greptime_mito_write_buffer_bytes{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum(sys_jemalloc_allocated{pod=~\"$datanode\"})", "instant": false, - "legendFormat": "{{instance}}", + "legendFormat": "allocated", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_resident{pod=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "resident", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_virtual_memory_bytes{pod=~\"$datanode\"})", + "hide": false, + "instant": false, + "legendFormat": "virtual-memory", + "range": true, + "refId": "C" } ], - "title": "Write buffer bytes", + "title": "Datanode Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -2626,6 +1717,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2661,8 +1753,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2670,23 +1761,27 @@ } ] }, - "unit": "none" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 61 + "h": 10, + "w": 9, + "x": 8, + "y": 24 }, - "id": 189, + "id": 233, "options": { "legend": { - "calcs": [], - "displayMode": "list", + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", "placement": "bottom", - "showLegend": true + "showLegend": true, + "sortBy": "Last *", + "sortDesc": false }, "tooltip": { "mode": "single", @@ -2697,27 +1792,49 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "sum by(instance) (greptime_mito_write_stall_total{instance=~\"$instance\"})", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum(sys_jemalloc_allocated{pod=~\"$frontend\"})", "instant": false, - "legendFormat": "{{instance}}", + "legendFormat": "allocated", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_resident{pod=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "resident", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_virtual_memory_bytes{pod=~\"$frontend\"})", + "hide": false, + "instant": false, + "legendFormat": "virtual-memory", + "range": true, + "refId": "C" } ], - "title": "Write buffer bytes", + "title": "Frontend Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -2725,6 +1842,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2760,8 +1878,240 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 7, + "x": 17, + "y": 24 + }, + "id": 235, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_allocated{pod=~\"$metasrv\"})", + "instant": false, + "legendFormat": "allocated", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(sys_jemalloc_resident{pod=~\"$metasrv\"})", + "hide": false, + "instant": false, + "legendFormat": "resident", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum(process_virtual_memory_bytes{pod=~\"$metasrv\"})", + "hide": false, + "instant": false, + "legendFormat": "virtual-memory", + "range": true, + "refId": "C" + } + ], + "title": "Metasrv Memory", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 192, + "panels": [], + "title": "Frontend APIs", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 202, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{pod=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "HTTP QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, { "color": "red", @@ -2777,20 +2127,18 @@ "h": 8, "w": 12, "x": 12, - "y": 61 + "y": 35 }, - "id": 170, + "id": 203, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2798,29 +2146,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{pod=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{instance}}-{{stage}}-p99", + "legendFormat": "[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Write stage", + "title": "HTTP P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -2828,6 +2170,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -2863,8 +2206,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -2876,28 +2218,25 @@ }, "overrides": [ { + "__systemRef": "hideSeriesFrom", "matcher": { - "id": "byRegexp", - "options": ".*?-bytes" + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[mycluster-frontend-5f94445cf8-mcmhf]-[/v1/prometheus/write]-[POST]-[204]-qps" + ], + "prefix": "All except:", + "readOnly": true + } }, "properties": [ { - "id": "custom.drawStyle", - "value": "points" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "custom.stacking", + "id": "custom.hideFrom", "value": { - "group": "A", - "mode": "none" + "legend": false, + "tooltip": false, + "viz": true } } ] @@ -2908,20 +2247,18 @@ "h": 8, "w": 12, "x": 0, - "y": 69 + "y": 43 }, - "id": 169, + "id": 211, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -2929,65 +2266,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum by(pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{instance}}-{{reason}}-success", + "legendFormat": "[{{pod}}]-[{{path}}]-[{{code}}]-qps", "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, reason) (rate(greptime_mito_flush_errors_total{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{reason}}-error", - "range": true, - "refId": "B", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance) (rate(greptime_mito_flush_bytes_total{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-bytes", - "range": true, - "refId": "C", - "useBackend": false + "refId": "A" } ], - "title": "Flush total", + "title": "gRPC QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -2995,6 +2290,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3030,306 +2326,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 69 - }, - "id": 191, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{stage}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$instance\", stage=\"total\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Read stage", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 77 - }, - "id": 172, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Compaction total", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3345,20 +2342,18 @@ "h": 8, "w": 12, "x": 12, - "y": 77 + "y": 43 }, - "id": 171, + "id": 212, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -3366,28 +2361,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{pod=~\"$frontend\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{instance}}-{{stage}}-p99", + "legendFormat": "[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Compaction stage", + "title": "gRPC P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -3395,6 +2385,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3430,8 +2421,942 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[mycluster-frontend-5c59b4cc9b-kpb6q]-qps" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 213, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod)(rate(greptime_servers_mysql_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "MySQL QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 214, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_quantile(0.99, sum by(pod) (rate(greptime_servers_mysql_query_elapsed_bucket{pod=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "MySQL P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "[mycluster-frontend-5f94445cf8-mcmhf]-[/v1/prometheus/write]-[POST]-[204]-qps" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 59 + }, + "id": 215, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(greptime_servers_postgres_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "PostgreSQL QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 59 + }, + "id": 216, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod) (rate(greptime_servers_postgres_query_elapsed_count{pod=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{pod}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "PostgreSQL P99 per Instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 217, + "panels": [], + "title": "Frontend <-> Datanode", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "rowsps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 218, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod)(rate(greptime_table_operator_ingest_rows{pod=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-rps", + "range": true, + "refId": "A" + } + ], + "title": "Ingest Rows per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 219, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, request_type) (rate(greptime_grpc_region_request_count{pod=~\"$frontend\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{request_type}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "Region Call QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 220, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, request_type) (rate(greptime_grpc_region_request_bucket{pod=~\"$frontend\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{request_type}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Region Call P99 per Instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 194, + "panels": [], + "title": "Mito Engine", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 83 + }, + "id": 201, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, type) (rate(greptime_mito_handle_request_elapsed_count{pod=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{type}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "Request QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 83 + }, + "id": 222, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{type}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Request P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" }, { "color": "red", @@ -3447,13 +3372,13 @@ "h": 8, "w": 12, "x": 0, - "y": 85 + "y": 91 }, - "id": 192, + "id": 200, "options": { "legend": { "calcs": [], - "displayMode": "list", + "displayMode": "table", "placement": "bottom", "showLegend": true }, @@ -3466,27 +3391,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "greptime_mito_cache_bytes{instance=~\"$instance\"}", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "greptime_mito_write_buffer_bytes{pod=~\"$datanode\"}", "instant": false, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "{{pod}}", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Cached bytes", + "title": "Write Buffer per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -3494,6 +3415,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3529,8 +3451,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3538,7 +3459,7 @@ } ] }, - "unit": "percentunit" + "unit": "decbytes" }, "overrides": [] }, @@ -3546,9 +3467,982 @@ "h": 8, "w": 12, "x": 12, - "y": 85 + "y": 91 }, - "id": 193, + "id": 221, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod) (greptime_mito_write_stall_total{pod=~\"$datanode\"})", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Write Stall per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 99 + }, + "id": 224, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, reason) (rate(greptime_mito_flush_requests_total{pod=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{reason}}]-success", + "range": true, + "refId": "A" + } + ], + "title": "Flush QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 99 + }, + "id": 229, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "greptime_mito_cache_bytes{pod=~\"$datanode\"}", + "instant": false, + "legendFormat": "{{pod}}-{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Cached Bytes per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 107 + }, + "id": 227, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(greptime_mito_read_stage_elapsed_count{pod=~\"$datanode\", stage=\"total\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}-p99", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 107 + }, + "id": 228, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{pod}}-{{stage}}-p99", + "range": true, + "refId": "A" + } + ], + "title": "Read Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 115 + }, + "id": 231, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod) (rate(greptime_mito_compaction_total_elapsed_count{pod=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "{{pod}}", + "range": true, + "refId": "A" + } + ], + "title": "Compaction OPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 115 + }, + "id": 230, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{pod}}]-compaction-p99", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 123 + }, + "id": 225, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{pod}}-{{stage}}-p99", + "range": true, + "refId": "A" + } + ], + "title": "Write Stage P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 123 + }, + "id": 232, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{pod=~\"$datanode\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "{{pod}}-{{stage}}-p99", + "range": true, + "refId": "A" + } + ], + "title": "Compaction P99 per Instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 131 + }, + "id": 195, + "panels": [], + "title": "OpenDAL", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 132 + }, + "id": 209, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, scheme, operation) (rate(opendal_requests_total{pod=~\"$datanode\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 142 + }, + "id": 196, "options": { "legend": { "calcs": [], @@ -3565,40 +4459,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, "editorMode": "code", - "expr": "sum(increase(greptime_mito_cache_hit[$__rate_interval])) by (instance, type) / (sum(increase(greptime_mito_cache_miss[$__rate_interval])) by (instance, type) + sum(increase(greptime_mito_cache_hit[$__rate_interval])) by (instance, type))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "expr": "sum by(pod, scheme) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\", operation=\"read\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{instance}}-{{type}}", + "legendFormat": "[{{pod}}]-[{{scheme}}]-qps", "range": true, - "refId": "A", - "useBackend": false + "refId": "A" } ], - "title": "Cache hit rate", + "title": "Read QPS per Instance", "type": "timeseries" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 93 - }, - "id": 173, - "panels": [], - "title": "OpenDAL", - "type": "row" - }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -3606,6 +4483,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -3641,8 +4519,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3652,352 +4529,24 @@ }, "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 94 - }, - "id": 178, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, scheme) (rate(opendal_requests_duration_seconds_bucket{instance=~\"$instance\",operation=\"read\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme) (rate(opendal_requests_duration_seconds_count{instance=~\"$instance\", operation=\"read\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{scheme}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Read requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 94 - }, - "id": 179, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, scheme) (rate(opendal_requests_duration_seconds_bucket{instance=~\"$instance\", operation=\"write\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme) (rate(opendal_requests_duration_seconds_count{instance=~\"$instance\", operation=\"write\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{scheme}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Write requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 6, + "h": 7, + "w": 12, "x": 12, - "y": 94 + "y": 142 }, - "id": 180, + "id": 198, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -4005,47 +4554,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, scheme) (rate(opendal_requests_duration_seconds_bucket{instance=~\"$instance\", operation=\"list\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\",operation=\"read\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-p99", + "legendFormat": "[{{pod}}]-{{scheme}}-p99", "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme) (rate(opendal_requests_duration_seconds_count{instance=~\"$instance\", operation=\"list\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{scheme}}-qps", - "range": true, - "refId": "B", - "useBackend": false + "refId": "A" } ], - "title": "List requests", + "title": "Read P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -4053,12 +4578,13 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "drawStyle": "points", + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { @@ -4088,8 +4614,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4097,205 +4622,26 @@ } ] }, - "unit": "s" + "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 94 - }, - "id": 182, - "options": { - "legend": { - "calcs": [ - "last" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, scheme) (rate(opendal_requests_duration_seconds_bucket{instance=~\"$instance\", operation=\"stat\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-p99", - "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme) (rate(opendal_requests_duration_seconds_count{instance=~\"$instance\", operation=\"stat\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{scheme}}-qps", - "range": true, - "refId": "B", - "useBackend": false - } - ], - "title": "Stat requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "points", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, + "h": 7, + "w": 12, "x": 0, - "y": 102 + "y": 149 }, - "id": 181, + "id": 199, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -4303,47 +4649,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, scheme, operation) (rate(opendal_requests_duration_seconds_bucket{instance=~\"$instance\", operation!~\"read|write|list\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "sum by(pod, scheme) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\", operation=\"write\"}[$__rate_interval]))", "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-{{operation}}-p99", + "legendFormat": "[{{pod}}]-[{{scheme}}]-qps", "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme, operation) (rate(opendal_requests_duration_seconds_count{instance=~\"$instance\", operation!~\"read|write|list\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{scheme}}-{{operation}}-qps", - "range": true, - "refId": "B", - "useBackend": false + "refId": "A" } ], - "title": "Requests duration", + "title": "Write QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -4351,6 +4673,7 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4386,8 +4709,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4397,54 +4719,24 @@ }, "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-bytes" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "bytes" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 102 + "h": 7, + "w": 12, + "x": 12, + "y": 149 }, - "id": 177, + "id": 204, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -4452,47 +4744,23 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(instance, le, scheme, operation) (rate(opendal_bytes_total_bucket{instance=~\"$instance\"}[$__rate_interval])))", - "fullMetaSearch": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\", operation=\"write\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-{{operation}}-p99", + "legendFormat": "[{{pod}}]-[{{scheme}}]-p99", "range": true, - "refId": "A", - "useBackend": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" - }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme, operation) (rate(opendal_bytes_total_count{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": false, - "interval": "$__rate_interval", - "legendFormat": "{{instance}}-{{scheme}}-{{operation}}-bytes", - "range": true, - "refId": "B", - "useBackend": false + "refId": "A" } ], - "title": "Bytes total", + "title": "Write P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, "fieldConfig": { "defaults": { @@ -4500,6 +4768,103 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 156 + }, + "id": 205, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, scheme) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "[{{pod}}]-[{{scheme}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "List QPS per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", @@ -4535,8 +4900,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4546,54 +4910,24 @@ }, "unit": "s" }, - "overrides": [ - { - "matcher": { - "id": "byRegexp", - "options": ".*?-qps" - }, - "properties": [ - { - "id": "custom.drawStyle", - "value": "line" - }, - { - "id": "custom.axisPlacement", - "value": "right" - }, - { - "id": "unit", - "value": "ops" - }, - { - "id": "custom.stacking", - "value": { - "group": "A", - "mode": "none" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 102 + "h": 7, + "w": 12, + "x": 12, + "y": 156 }, - "id": 176, + "id": 206, "options": { "legend": { - "calcs": [ - "last" - ], + "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, @@ -4601,42 +4935,226 @@ { "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "disableTextWrap": false, - "editorMode": "builder", - "exemplar": false, - "expr": "sum by(instance, scheme, operation) (rate(opendal_requests_total{instance=~\"$instance\"}[$__rate_interval]))", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, scheme) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", "instant": false, - "legendFormat": "{{instance}}-{{scheme}}-{{operation}}-qps", + "interval": "", + "legendFormat": "[{{pod}}]-[{{scheme}}]-p99", "range": true, - "refId": "B", - "useBackend": false + "refId": "A" } ], - "title": "Requests total", + "title": "List P99 per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 163 + }, + "id": 207, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "sum by(pod, scheme, operation) (rate(opendal_requests_duration_seconds_count{pod=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-qps", + "range": true, + "refId": "A" + } + ], + "title": "Other Requests per Instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "points", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 3, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 163 + }, + "id": 210, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by(pod, le, scheme, operation) (rate(opendal_requests_duration_seconds_bucket{pod=~\"$datanode\", operation!~\"read|write|list\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "[{{pod}}]-[{{scheme}}]-[{{operation}}]-p99", + "range": true, + "refId": "A" + } + ], + "title": "Other Request P99 per Instance", "type": "timeseries" } ], - "refresh": "5s", - "schemaVersion": 38, - "style": "dark", + "refresh": "", + "schemaVersion": 39, "tags": [], "templating": { "list": [ { "current": { "selected": false, - "text": "prometheus-1", - "value": "prometheus-1" + "text": "metrics", + "value": "P177A7EA3611FE6B1" }, "hide": 0, "includeAll": false, "multi": false, - "name": "datasource", + "name": "metrics", "options": [], "query": "prometheus", "queryValue": "", @@ -4646,19 +5164,185 @@ "type": "datasource" }, { - "current": {}, + "current": { + "selected": false, + "text": "information_schema", + "value": "P0CE5E4D2C4819379" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "information_schema", + "options": [], + "query": "mysql", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, "datasource": { "type": "prometheus", - "uid": "${DS_PROMETHEUS-1}" + "uid": "${metrics}" }, - "definition": "label_values(instance)", + "definition": "label_values(greptime_app_version{app=~\"greptime-datanode|greptime-frontend|greptime-metasrv|greptime-flownode\"},app)", "hide": 0, "includeAll": true, - "multi": false, - "name": "instance", + "multi": true, + "name": "roles", "options": [], "query": { - "query": "label_values(instance)", + "qryType": 1, + "query": "label_values(greptime_app_version{app=~\"greptime-datanode|greptime-frontend|greptime-metasrv|greptime-flownode\"},app)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=~\"$role\"},pod)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "pods", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=~\"$role\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-datanode\"},pod)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "datanode", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-datanode\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-frontend\"},pod)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "frontend", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-frontend\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-metasrv\"},pod)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "metasrv", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-metasrv\"},pod)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${metrics}" + }, + "definition": "label_values(greptime_app_version{app=\"greptime-flownode\"},pod)", + "hide": 2, + "includeAll": true, + "multi": true, + "name": "flownode", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(greptime_app_version{app=\"greptime-flownode\"},pod)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, @@ -4670,13 +5354,13 @@ ] }, "time": { - "from": "now-30m", + "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "GreptimeDB-Cluster", - "uid": "ea35efe5-918e-44fa-9743-e9aa1a340a3f", - "version": 11, + "title": "GreptimeDB Cluster Metrics", + "uid": "ce3q6xwn3xa0wa", + "version": 1, "weekStart": "" -} \ No newline at end of file +}