{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "The Grafana dashboards for GreptimeDB.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": 34, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 279, "panels": [], "title": "Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "The start time of GreptimeDB.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "fieldMinMax": false, "mappings": [], "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 3, "x": 0, "y": 1 }, "id": 265, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "limit": 1, "values": true }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "time() - process_start_time_seconds", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], "title": "Uptime", "type": "stat" }, { "datasource": { "type": "mysql", "uid": "${information_schema}" }, "description": "GreptimeDB version.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 2, "x": 3, "y": 1 }, "id": 239, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "/^pkg_version$/", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT pkg_version FROM information_schema.build_info", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Version", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Total ingestion rate.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "fieldMinMax": false, "mappings": [], "max": 2, "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "rowsps" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 5, "y": 1 }, "id": 249, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Total Ingestion Rate", "type": "stat" }, { "datasource": { "type": "mysql", "uid": "${information_schema}" }, "description": "Total number of data file size.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 9, "y": 1 }, "id": 248, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "select SUM(disk_size) from information_schema.region_statistics;", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Total Storage Size", "type": "stat" }, { "datasource": { "type": "mysql", "uid": "${information_schema}" }, "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "sishort" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 13, "y": 1 }, "id": 254, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "select SUM(region_rows) from information_schema.region_statistics;", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Total Rows", "type": "stat" }, { "datasource": { "type": "mysql", "uid": "${information_schema}" }, "description": "The deployment topology of GreptimeDB.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 0, "y": 5 }, "id": 243, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';", "refId": "datanode", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';", "refId": "frontend", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';", "refId": "metasrv", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';", "refId": "flownode", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Deployment", "type": "stat" }, { "datasource": { "type": "mysql", "uid": "${information_schema}" }, "description": "The number of the key resources in GreptimeDB.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 5, "y": 5 }, "id": 247, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')", "refId": "databases", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "rawQuery": true, "rawSql": "SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'", "refId": "tables", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT COUNT(region_id) as regions FROM information_schema.region_peers", "refId": "A", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT COUNT(*) as flows FROM information_schema.flows", "refId": "B", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Database Resources", "type": "stat" }, { "datasource": { "type": "mysql", "uid": "${information_schema}" }, "description": "The data size of wal/index/manifest in the GreptimeDB.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 5, "x": 10, "y": 5 }, "id": 278, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.6.0", "targets": [ { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;\n", "refId": "WAL", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT SUM(index_size) as index FROM information_schema.region_statistics;\n", "refId": "Index", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } }, { "dataset": "information_schema", "datasource": { "type": "mysql", "uid": "${information_schema}" }, "editorMode": "code", "format": "table", "hide": false, "rawQuery": true, "rawSql": "SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;\n", "refId": "manifest", "sql": { "columns": [ { "parameters": [], "type": "function" } ], "groupBy": [ { "property": { "type": "string" }, "type": "groupBy" } ], "limit": 50 } } ], "title": "Data Size", "type": "stat" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 275, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "rowsps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 18 }, "id": 193, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "ingestion", "range": true, "refId": "C" } ], "title": "Total Ingestion Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "rowsps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 70 }, "id": 277, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "http-logs", "range": true, "refId": "http_logs" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "prometheus-remote-write", "range": true, "refId": "prometheus-remote-write" } ], "title": "Ingestion Rate by Type", "type": "timeseries" } ], "title": "Ingestion", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 276, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 19 }, "id": 255, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", "instant": false, "legendFormat": "mysql", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "pg", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~\"$frontend\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "promql", "range": true, "refId": "C" } ], "title": "Total Query Rate", "type": "timeseries" } ], "title": "Queries", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 274, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current memory usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 20 }, "id": 256, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)", "instant": false, "legendFormat": "[{{instance}}]-[{{ pod }}]", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$datanode\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Datanode Memory per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current cpu usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 20 }, "id": 262, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$datanode\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Datanode CPU Usage per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current memory usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 30 }, "id": 266, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$frontend\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Frontend Memory per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current cpu usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 30 }, "id": 268, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$frontend\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Frontend CPU Usage per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current memory usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 40 }, "id": 269, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$metasrv\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Metasrv Memory per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current cpu usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 40 }, "id": 271, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$metasrv\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Metasrv CPU Usage per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current memory usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 0, "y": 50 }, "id": 272, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$flownode\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Flownode Memory per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Current cpu usage by instance", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 10, "w": 12, "x": 12, "y": 50 }, "id": 273, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Mean", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$flownode\"})", "hide": false, "instant": false, "legendFormat": "limit", "range": true, "refId": "B" } ], "title": "Flownode CPU Usage per Instance", "type": "timeseries" } ], "title": "Resources", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 280, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "HTTP QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 61 }, "id": 281, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]", "range": true, "refId": "A" } ], "title": "HTTP QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "HTTP P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 61 }, "id": 282, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, "refId": "A" } ], "title": "HTTP P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "gRPC QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 69 }, "id": 283, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]", "range": true, "refId": "A" } ], "title": "gRPC QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "gRPC P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 69 }, "id": 284, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99", "range": true, "refId": "A" } ], "title": "gRPC P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "MySQL QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 77 }, "id": 285, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "MySQL QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "MySQL P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 77 }, "id": 286, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "exemplar": false, "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{ instance }}]-[{{ pod }}]-p99", "range": true, "refId": "A" } ], "title": "MySQL P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "PostgreSQL QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 85 }, "id": 287, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "PostgreSQL QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "PostgreSQL P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 85 }, "id": 288, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "A" } ], "title": "PostgreSQL P99 per Instance", "type": "timeseries" } ], "title": "Frontend Requests", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 }, "id": 289, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Ingestion rate by row as in each frontend", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "rowsps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 62 }, "id": 292, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Ingest Rows per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Region Call QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 68 }, "id": 290, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~\"$frontend\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", "range": true, "refId": "A" } ], "title": "Region Call QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Region Call P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 68 }, "id": 291, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~\"$frontend\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]", "range": true, "refId": "A" } ], "title": "Region Call P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Per-stage time for frontend to handle bulk insert requests", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "[minipc-1:4000]-[]-[encode]-AVG", "[minipc-1:4000]-[]-[partition]-AVG", "[minipc-1:4000]-[]-[decode_request]-AVG", "[minipc-1:4000]-[]-[datanode_handle]-AVG" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 76 }, "id": 336, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", "range": true, "refId": "B" } ], "title": "Frontend Handle Bulk Insert Elapsed Time ", "type": "timeseries" } ], "title": "Frontend to Datanode", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, "id": 293, "panels": [], "title": "Mito Engine", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Request QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 294, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" } ], "title": "Request OPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Request P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, "id": 295, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" } ], "title": "Request P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write Buffer per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "id": 296, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_mito_write_buffer_bytes{instance=~\"$datanode\"}", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Write Buffer per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Ingestion size by row counts.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "rowsps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, "id": 297, "options": { "legend": { "calcs": [ "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Write Rows per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Flush QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, "id": 298, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]", "range": true, "refId": "A" } ], "title": "Flush OPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write Stall per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, "id": 299, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Write Stall per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Read Stage OPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 39 }, "id": 300, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\", stage=\"total\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Read Stage OPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Read Stage P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 39 }, "id": 301, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", "range": true, "refId": "A" } ], "title": "Read Stage P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write Stage P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 47 }, "id": 302, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]", "range": true, "refId": "A" } ], "title": "Write Stage P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Compaction OPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 47 }, "id": 303, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{ instance }}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Compaction OPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Compaction latency by stage", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 55 }, "id": 304, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg", "range": true, "refId": "B" } ], "title": "Compaction Elapsed Time per Instance by Stage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Compaction P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 55 }, "id": 305, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction", "range": true, "refId": "A" } ], "title": "Compaction P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 63 }, "id": 306, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-throughput", "range": true, "refId": "C" } ], "title": "WAL write size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Cached Bytes per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 63 }, "id": 307, "options": { "legend": { "calcs": [], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_mito_cache_bytes{instance=~\"$datanode\"}", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" } ], "title": "Cached Bytes per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Ongoing compaction task count", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 71 }, "id": 308, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_mito_inflight_compaction_count", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Inflight Compaction", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Raft engine (local disk) log store sync latency, p99", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 71 }, "id": 310, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "A" } ], "title": "WAL sync duration seconds", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write-ahead log operations latency at p99", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 79 }, "id": 311, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99", "range": true, "refId": "A" } ], "title": "Log Store op duration seconds", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Ongoing flush task count", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 79 }, "id": 312, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_mito_inflight_flush_count", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]", "range": true, "refId": "A" } ], "title": "Inflight Flush", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Compaction oinput output bytes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 87 }, "id": 335, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-input", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-output", "range": true, "refId": "B" } ], "title": "Compaction Input/Output Bytes", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Per-stage elapsed time for region worker to handle bulk insert region requests.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "__systemRef": "hideSeriesFrom", "matcher": { "id": "byNames", "options": { "mode": "exclude", "names": [ "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG", "[minipc-1:4000]-[]-[bulk_extend]-AVG", "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG", "[minipc-1:4000]-[]-[process_bulk_req]-AVG", "[minipc-1:4000]-[]-[write_bulk]-AVG" ], "prefix": "All except:", "readOnly": true } }, "properties": [ { "id": "custom.hideFrom", "value": { "legend": false, "tooltip": false, "viz": true } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 87 }, "id": 337, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", "range": true, "refId": "B" } ], "title": "Region Worker Handle Bulk Insert Requests", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Compaction oinput output bytes", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 95 }, "id": 348, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (greptime_mito_memtable_active_series_count)", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-series", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod) (greptime_mito_memtable_field_builder_count)", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-field_builders", "range": true, "refId": "B" } ], "title": "Active Series and Field Builders Count", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Per-stage elapsed time for region worker to decode requests.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 95 }, "id": 338, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "builder", "expr": "histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-P95", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG", "range": true, "refId": "B" } ], "title": "Region Worker Convert Requests", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "The local cache miss of the datanode.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 103 }, "id": 349, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~\"$datanode\"}[$__rate_interval]))", "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]", "range": true, "refId": "A" } ], "title": "Cache Miss", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 111 }, "id": 313, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 10, "w": 24, "x": 0, "y": 232 }, "id": 314, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Read QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 242 }, "id": 315, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "Read QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Read P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 242 }, "id": 316, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=~\"read|Reader::read\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "Read P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 249 }, "id": 317, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "Write QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Write P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 249 }, "id": 318, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "Write P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "List QPS per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 256 }, "id": 319, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))", "instant": false, "interval": "", "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", "range": true, "refId": "A" } ], "title": "List QPS per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "List P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 256 }, "id": 320, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))", "instant": false, "interval": "", "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]", "range": true, "refId": "A" } ], "title": "List P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Other Requests per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 263 }, "id": 321, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "Other Requests per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Other Request P99 per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 3, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 263 }, "id": 322, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|write|list|Writer::write|Writer::close|Reader::read\"}[$__rate_interval])))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A" } ], "title": "Other Request P99 per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Total traffic as in bytes by instance and operation", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "decbytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 270 }, "id": 323, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]", "range": true, "refId": "A", "useBackend": false } ], "title": "Opendal traffic", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "OpenDAL error counts per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 270 }, "id": 334, "options": { "legend": { "calcs": [ "lastNotNull" ], "displayMode": "table", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "multi", "sort": "desc" } }, "pluginVersion": "12.0.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\", error!=\"NotFound\"}[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]", "range": true, "refId": "A" } ], "title": "OpenDAL errors per Instance", "type": "timeseries" } ], "title": "OpenDAL", "type": "row" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 112 }, "id": 351, "panels": [], "title": "Remote WAL", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Triggered region flush total", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 113 }, "id": 350, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "meta_triggered_region_flush_total", "instant": false, "legendFormat": "{{pod}}-{{topic_name}}", "range": true, "refId": "A" } ], "title": "Triggered region flush total", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Triggered region checkpoint total", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 113 }, "id": 352, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "meta_triggered_region_checkpoint_total", "instant": false, "legendFormat": "{{pod}}-{{topic_name}}", "range": true, "refId": "A" } ], "title": "Triggered region checkpoint total", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Topic estimated max replay size", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 121 }, "id": 353, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "meta_topic_estimated_replay_size", "instant": false, "legendFormat": "{{pod}}-{{topic_name}}", "range": true, "refId": "A" } ], "title": "Topic estimated replay size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Kafka logstore's bytes traffic", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 121 }, "id": 354, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])", "instant": false, "legendFormat": "{{pod}}-{{logstore}}", "range": true, "refId": "A" } ], "title": "Kafka logstore's bytes traffic", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 129 }, "id": 324, "panels": [], "title": "Metasrv", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Counter of region migration by source and destination", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "axisPlacement": "auto", "fillOpacity": 70, "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineWidth": 1 }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 130 }, "id": 325, "options": { "colWidth": 0.9, "legend": { "displayMode": "list", "placement": "bottom", "showLegend": false }, "rowHeight": 0.9, "showValue": "auto", "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_meta_region_migration_stat{datanode_type=\"src\"}", "instant": false, "legendFormat": "from-datanode-{{datanode_id}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_meta_region_migration_stat{datanode_type=\"desc\"}", "hide": false, "instant": false, "legendFormat": "to-datanode-{{datanode_id}}", "range": true, "refId": "B" } ], "title": "Region migration datanode", "type": "status-history" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Counter of region migration error", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 130 }, "id": 326, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_meta_region_migration_error", "instant": false, "legendFormat": "{{pod}}-{{state}}-{{error_type}}", "range": true, "refId": "A" } ], "title": "Region migration error", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 138 }, "id": 327, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_datanode_load", "instant": false, "legendFormat": "Datanode-{{datanode_id}}-writeload", "range": true, "refId": "A" } ], "title": "Datanode load", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Displays the rate of SQL executions processed by the Meta service using the RDS backend.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 138 }, "id": 339, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])", "instant": false, "legendFormat": "{{pod}} {{op}} {{type}} {{result}} ", "range": true, "refId": "A" } ], "title": "Rate of SQL Executions (RDS)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Measures the response time of SQL executions via the RDS backend. ", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 146 }, "id": 340, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{pod}} {{op}} {{type}} {{result}} p90", "range": true, "refId": "A", "useBackend": false } ], "title": "SQL Execution Latency (RDS)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 146 }, "id": 341, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.90, sum by(pod, le, name) (\n rate(greptime_meta_handler_execute_bucket[$__rate_interval])\n))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{pod}} {{name}} p90", "range": true, "refId": "A", "useBackend": false } ], "title": "Handler Execution Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.\n", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 154 }, "id": 342, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Heartbeat Packet Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 154 }, "id": 345, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "rate(greptime_meta_heartbeat_rate[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{pod}}", "range": true, "refId": "A", "useBackend": false } ], "title": "Meta Heartbeat Receive Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 162 }, "id": 343, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{pod}}-{{op}} p99", "range": true, "refId": "A", "useBackend": false } ], "title": "Meta KV Ops Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 162 }, "id": 346, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "{{pod}}-{{op}} p99", "range": true, "refId": "A", "useBackend": false } ], "title": "Rate of meta KV Ops", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 170 }, "id": 347, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, "legendFormat": "CreateLogicalTables-{{step}} p90", "range": true, "refId": "A", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "CreateTable-{{step}} p90", "range": true, "refId": "B", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "CreateView-{{step}} p90", "range": true, "refId": "C", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "CreateFlow-{{step}} p90", "range": true, "refId": "D", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "DropTable-{{step}} p90", "range": true, "refId": "E", "useBackend": false }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "disableTextWrap": false, "editorMode": "code", "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, "instant": false, "legendFormat": "AlterTable-{{step}} p90", "range": true, "refId": "F", "useBackend": false } ], "title": "DDL Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Reconciliation stats", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 170 }, "id": 355, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_meta_reconciliation_stats", "hide": false, "instant": false, "legendFormat": "{{pod}}-{{table_type}}-{{type}}", "range": true, "refId": "A" } ], "title": "Reconciliation stats", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Elapsed of Reconciliation steps ", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 178 }, "id": 356, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.6.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)", "hide": false, "instant": false, "legendFormat": "{{procedure_name}}-{{step}}-P90", "range": true, "refId": "A" } ], "title": "Reconciliation steps", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 186 }, "id": 328, "panels": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Flow Ingest / Output Rate.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3055 }, "id": 329, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))", "instant": false, "legendFormat": "[{{pod}}]-[{{instance}}]-[{{direction}}]", "range": true, "refId": "A" } ], "title": "Flow Ingest / Output Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Flow Ingest Latency.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 3055 }, "id": 330, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-p95", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-p99", "range": true, "refId": "B" } ], "title": "Flow Ingest Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Flow Operation Latency.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "points", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 9, "x": 0, "y": 3063 }, "id": 331, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))", "hide": false, "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99", "range": true, "refId": "B" } ], "title": "Flow Operation Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Flow Buffer Size per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 9, "x": 9, "y": 3063 }, "id": 332, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "greptime_flow_input_buf_size", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}]", "range": true, "refId": "A" } ], "title": "Flow Buffer Size per Instance", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "description": "Flow Processing Error per Instance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 6, "x": 18, "y": 3063 }, "id": 333, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${metrics}" }, "editorMode": "code", "expr": "sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))", "instant": false, "legendFormat": "[{{instance}}]-[{{pod}}]-[{{code}}]", "range": true, "refId": "A" } ], "title": "Flow Processing Error per Instance", "type": "timeseries" } ], "title": "Flownode", "type": "row" } ], "preload": false, "refresh": "5s", "schemaVersion": 41, "tags": [], "templating": { "list": [ { "current": { "text": "metrics", "value": "P177A7EA3611FE6B1" }, "includeAll": false, "name": "metrics", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "current": { "text": "information_schema", "value": "P0CE5E4D2C4819379" }, "includeAll": false, "name": "information_schema", "options": [], "query": "mysql", "refresh": 1, "regex": "", "type": "datasource" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${metrics}" }, "definition": "label_values(greptime_app_version{app=\"greptime-datanode\"},instance)", "hide": 2, "includeAll": true, "multi": true, "name": "datanode", "options": [], "query": { "qryType": 1, "query": "label_values(greptime_app_version{app=\"greptime-datanode\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${metrics}" }, "definition": "label_values(greptime_app_version{app=\"greptime-frontend\"},instance)", "hide": 2, "includeAll": true, "multi": true, "name": "frontend", "options": [], "query": { "qryType": 1, "query": "label_values(greptime_app_version{app=\"greptime-frontend\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${metrics}" }, "definition": "label_values(greptime_app_version{app=\"greptime-metasrv\"},instance)", "hide": 2, "includeAll": true, "multi": true, "name": "metasrv", "options": [], "query": { "qryType": 1, "query": "label_values(greptime_app_version{app=\"greptime-metasrv\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" }, { "current": { "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${metrics}" }, "definition": "label_values(greptime_app_version{app=\"greptime-flownode\"},instance)", "hide": 2, "includeAll": true, "multi": true, "name": "flownode", "options": [], "query": { "qryType": 1, "query": "label_values(greptime_app_version{app=\"greptime-flownode\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "type": "query" } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "GreptimeDB", "uid": "dejf3k5e7g2kgb", "version": 15 }