diff --git a/Cargo.lock b/Cargo.lock
index 6f9fcb4ae8..c5ab8f80f3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -213,7 +213,7 @@ checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c"
[[package]]
name = "api"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arrow-schema 58.3.0",
"common-base",
@@ -936,7 +936,7 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
[[package]]
name = "auth"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -1574,7 +1574,7 @@ dependencies = [
[[package]]
name = "cache"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"catalog",
"common-error",
@@ -1610,7 +1610,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "catalog"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arrow 58.3.0",
@@ -1945,7 +1945,7 @@ checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
[[package]]
name = "cli"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-stream",
"async-trait",
@@ -2004,7 +2004,7 @@ dependencies = [
[[package]]
name = "client"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arc-swap",
@@ -2036,7 +2036,7 @@ dependencies = [
"serde_json",
"snafu 0.8.6",
"store-api",
- "substrait 1.1.1",
+ "substrait 1.1.2",
"tokio",
"tokio-stream",
"tonic 0.14.2",
@@ -2085,7 +2085,7 @@ dependencies = [
[[package]]
name = "cmd"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -2235,7 +2235,7 @@ dependencies = [
[[package]]
name = "common-base"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"anymap2",
@@ -2255,14 +2255,14 @@ dependencies = [
[[package]]
name = "common-catalog"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"const_format",
]
[[package]]
name = "common-config"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-base",
"common-error",
@@ -2286,7 +2286,7 @@ dependencies = [
[[package]]
name = "common-datasource"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arrow 58.3.0",
"arrow-schema 58.3.0",
@@ -2322,7 +2322,7 @@ dependencies = [
[[package]]
name = "common-decimal"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"bigdecimal 0.4.8",
"common-error",
@@ -2335,7 +2335,7 @@ dependencies = [
[[package]]
name = "common-error"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-macro",
"http 1.3.1",
@@ -2346,7 +2346,7 @@ dependencies = [
[[package]]
name = "common-event-recorder"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -2369,7 +2369,7 @@ dependencies = [
[[package]]
name = "common-frontend"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -2390,7 +2390,7 @@ dependencies = [
[[package]]
name = "common-function"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"api",
@@ -2453,7 +2453,7 @@ dependencies = [
[[package]]
name = "common-greptimedb-telemetry"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"common-runtime",
@@ -2470,7 +2470,7 @@ dependencies = [
[[package]]
name = "common-grpc"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arrow-flight",
@@ -2505,7 +2505,7 @@ dependencies = [
[[package]]
name = "common-grpc-expr"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"common-base",
@@ -2525,7 +2525,7 @@ dependencies = [
[[package]]
name = "common-macro"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"greptime-proto",
"once_cell",
@@ -2536,7 +2536,7 @@ dependencies = [
[[package]]
name = "common-mem-prof"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"anyhow",
"common-error",
@@ -2552,7 +2552,7 @@ dependencies = [
[[package]]
name = "common-memory-manager"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-error",
"common-macro",
@@ -2564,7 +2564,7 @@ dependencies = [
[[package]]
name = "common-meta"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"anymap2",
"api",
@@ -2635,7 +2635,7 @@ dependencies = [
[[package]]
name = "common-options"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-grpc",
"humantime-serde",
@@ -2645,11 +2645,11 @@ dependencies = [
[[package]]
name = "common-plugins"
-version = "1.1.1"
+version = "1.1.2"
[[package]]
name = "common-pprof"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-error",
"common-macro",
@@ -2660,7 +2660,7 @@ dependencies = [
[[package]]
name = "common-procedure"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-stream",
@@ -2689,7 +2689,7 @@ dependencies = [
[[package]]
name = "common-procedure-test"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"common-procedure",
@@ -2699,7 +2699,7 @@ dependencies = [
[[package]]
name = "common-query"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -2729,7 +2729,7 @@ dependencies = [
[[package]]
name = "common-recordbatch"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arc-swap",
"common-base",
@@ -2754,7 +2754,7 @@ dependencies = [
[[package]]
name = "common-runtime"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"clap",
@@ -2783,7 +2783,7 @@ dependencies = [
[[package]]
name = "common-session"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"serde",
"strum 0.27.1",
@@ -2791,7 +2791,7 @@ dependencies = [
[[package]]
name = "common-sql"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arrow-schema 58.3.0",
"common-base",
@@ -2811,7 +2811,7 @@ dependencies = [
[[package]]
name = "common-stat"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-base",
"common-runtime",
@@ -2826,7 +2826,7 @@ dependencies = [
[[package]]
name = "common-telemetry"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"backtrace",
"common-base",
@@ -2855,7 +2855,7 @@ dependencies = [
[[package]]
name = "common-test-util"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"client",
"common-grpc",
@@ -2868,7 +2868,7 @@ dependencies = [
[[package]]
name = "common-time"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arrow 58.3.0",
"chrono",
@@ -2886,7 +2886,7 @@ dependencies = [
[[package]]
name = "common-version"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"cargo-manifest",
"const_format",
@@ -2896,7 +2896,7 @@ dependencies = [
[[package]]
name = "common-wal"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-base",
"common-error",
@@ -2919,7 +2919,7 @@ dependencies = [
[[package]]
name = "common-workload"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"common-telemetry",
"serde",
@@ -4360,7 +4360,7 @@ dependencies = [
[[package]]
name = "datanode"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arrow-flight",
@@ -4434,7 +4434,7 @@ checksum = "c286de4e81ea2590afc24d754e0f83810c566f50a1388fa75ebd57928c0d9745"
[[package]]
name = "datatypes"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arrow 58.3.0",
"arrow-array 58.3.0",
@@ -5171,7 +5171,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "file-engine"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -5302,7 +5302,7 @@ checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8"
[[package]]
name = "flow"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arrow 58.3.0",
@@ -5371,7 +5371,7 @@ dependencies = [
"sql",
"store-api",
"strum 0.27.1",
- "substrait 1.1.1",
+ "substrait 1.1.2",
"table",
"tokio",
"tokio-stream",
@@ -5454,7 +5454,7 @@ checksum = "28dd6caf6059519a65843af8fe2a3ae298b14b80179855aeb4adc2c1934ee619"
[[package]]
name = "frontend"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arc-swap",
@@ -6745,7 +6745,7 @@ dependencies = [
[[package]]
name = "index"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"asynchronous-codec",
@@ -7844,7 +7844,7 @@ checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
[[package]]
name = "log-query"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"chrono",
"common-error",
@@ -7856,7 +7856,7 @@ dependencies = [
[[package]]
name = "log-store"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-stream",
"async-trait",
@@ -8175,7 +8175,7 @@ dependencies = [
[[package]]
name = "meta-client"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -8207,7 +8207,7 @@ dependencies = [
[[package]]
name = "meta-srv"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -8307,7 +8307,7 @@ dependencies = [
[[package]]
name = "metric-engine"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"aquamarine",
@@ -8408,7 +8408,7 @@ dependencies = [
[[package]]
name = "mito-codec"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"bytes",
@@ -8433,7 +8433,7 @@ dependencies = [
[[package]]
name = "mito2"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"aquamarine",
@@ -9222,7 +9222,7 @@ dependencies = [
[[package]]
name = "object-store"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"anyhow",
"async-trait",
@@ -9777,7 +9777,7 @@ dependencies = [
[[package]]
name = "operator"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"api",
@@ -9836,7 +9836,7 @@ dependencies = [
"sql",
"sqlparser",
"store-api",
- "substrait 1.1.1",
+ "substrait 1.1.2",
"table",
"tokio",
"tokio-util",
@@ -10121,7 +10121,7 @@ checksum = "e3c406c9e2aa74554e662d2c2ee11cd3e73756988800be7e6f5eddb16fed4699"
[[package]]
name = "partition"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"async-trait",
@@ -10515,7 +10515,7 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pipeline"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"api",
@@ -10672,7 +10672,7 @@ dependencies = [
[[package]]
name = "plugins"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"auth",
"catalog",
@@ -11012,7 +11012,7 @@ dependencies = [
[[package]]
name = "promql"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"async-trait",
@@ -11364,7 +11364,7 @@ dependencies = [
[[package]]
name = "puffin"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-compression",
"async-trait",
@@ -11426,7 +11426,7 @@ dependencies = [
[[package]]
name = "query"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"api",
@@ -11496,7 +11496,7 @@ dependencies = [
"sql",
"sqlparser",
"store-api",
- "substrait 1.1.1",
+ "substrait 1.1.2",
"table",
"tokio",
"tokio-stream",
@@ -13068,7 +13068,7 @@ dependencies = [
[[package]]
name = "servers"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"api",
@@ -13206,7 +13206,7 @@ dependencies = [
[[package]]
name = "session"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"ahash 0.8.12",
"api",
@@ -13559,7 +13559,7 @@ dependencies = [
[[package]]
name = "sql"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arrow-buffer 58.3.0",
@@ -13620,7 +13620,7 @@ dependencies = [
[[package]]
name = "sqlness-runner"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"clap",
@@ -13900,7 +13900,7 @@ dependencies = [
[[package]]
name = "standalone"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"catalog",
@@ -13944,7 +13944,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "store-api"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"aquamarine",
@@ -14137,7 +14137,7 @@ dependencies = [
[[package]]
name = "substrait"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"async-trait",
"bytes",
@@ -14259,7 +14259,7 @@ dependencies = [
[[package]]
name = "table"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arc-swap",
@@ -14542,7 +14542,7 @@ checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683"
[[package]]
name = "tests-fuzz"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"arbitrary",
"async-trait",
@@ -14587,7 +14587,7 @@ dependencies = [
[[package]]
name = "tests-integration"
-version = "1.1.1"
+version = "1.1.2"
dependencies = [
"api",
"arrow-flight",
@@ -14667,7 +14667,7 @@ dependencies = [
"sqlx",
"standalone",
"store-api",
- "substrait 1.1.1",
+ "substrait 1.1.2",
"table",
"tempfile",
"time",
diff --git a/Cargo.toml b/Cargo.toml
index 60e29c40de..29cbc47111 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -75,7 +75,7 @@ members = [
resolver = "2"
[workspace.package]
-version = "1.1.1"
+version = "1.1.2"
edition = "2024"
license = "Apache-2.0"
diff --git a/grafana/dashboards/metrics/cluster/dashboard.json b/grafana/dashboards/metrics/cluster/dashboard.json
index 01253bd83b..507d9312e6 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.json
+++ b/grafana/dashboards/metrics/cluster/dashboard.json
@@ -36,7 +36,7 @@
"x": 0,
"y": 0
},
- "id": 279,
+ "id": 393,
"panels": [],
"title": "Overview",
"type": "row"
@@ -68,12 +68,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 3,
"x": 0,
- "y": 1
+ "y": 1,
+ "w": 3,
+ "h": 4
},
- "id": 265,
+ "id": 380,
"options": {
"colorMode": "value",
"graphMode": "none",
@@ -142,12 +142,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 2,
"x": 3,
- "y": 1
+ "y": 1,
+ "w": 3,
+ "h": 4
},
- "id": 239,
+ "id": 381,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -227,12 +227,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
+ "x": 6,
+ "y": 1,
"w": 4,
- "x": 5,
- "y": 1
+ "h": 4
},
- "id": 249,
+ "id": 382,
"options": {
"colorMode": "value",
"graphMode": "none",
@@ -270,16 +270,18 @@
},
{
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
- "description": "Total number of data file size.",
+ "description": "Total query API call rate across MySQL, PostgreSQL, and PromQL frontends.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
+ "fieldMinMax": false,
"mappings": [],
+ "max": 2,
"thresholds": {
"mode": "absolute",
"steps": [
@@ -288,20 +290,20 @@
}
]
},
- "unit": "decbytes"
+ "unit": "reqps"
},
"overrides": []
},
"gridPos": {
- "h": 4,
+ "x": 10,
+ "y": 1,
"w": 4,
- "x": 9,
- "y": 1
+ "h": 4
},
- "id": 248,
+ "id": 383,
"options": {
"colorMode": "value",
- "graphMode": "area",
+ "graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
@@ -310,7 +312,8 @@
"lastNotNull"
],
"fields": "",
- "values": false
+ "limit": 1,
+ "values": true
},
"showPercentChange": false,
"textMode": "auto",
@@ -319,72 +322,62 @@
"pluginVersion": "11.6.0",
"targets": [
{
- "dataset": "information_schema",
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
"editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "select SUM(disk_size) from information_schema.region_statistics;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- }
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "queries",
+ "range": true,
+ "refId": "A"
}
],
- "title": "Total Storage Size",
+ "title": "Total Query Rate",
"type": "stat"
},
{
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
- "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.",
+ "description": "Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
+ "fieldMinMax": false,
"mappings": [],
+ "max": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
}
]
},
- "unit": "sishort"
+ "unit": "eps"
},
"overrides": []
},
"gridPos": {
- "h": 4,
+ "x": 14,
+ "y": 1,
"w": 4,
- "x": 13,
- "y": 1
+ "h": 4
},
- "id": 254,
+ "id": 384,
"options": {
"colorMode": "value",
- "graphMode": "area",
+ "graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
@@ -393,7 +386,8 @@
"lastNotNull"
],
"fields": "",
- "values": false
+ "limit": 1,
+ "values": true
},
"showPercentChange": false,
"textMode": "auto",
@@ -402,36 +396,94 @@
"pluginVersion": "11.6.0",
"targets": [
{
- "dataset": "information_schema",
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
"editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "select SUM(region_rows) from information_schema.region_statistics;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- }
+ "expr": "sum(rate(greptime_servers_error{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "errors",
+ "range": true,
+ "refId": "A"
}
],
- "title": "Total Rows",
+ "title": "User-facing Error Rate",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Process restarts over the selected time range across GreptimeDB roles.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "max": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 18,
+ "y": 1,
+ "w": 6,
+ "h": 4
+ },
+ "id": 385,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "limit": 1,
+ "values": true
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(changes(process_start_time_seconds[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "restarts",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Recent Restarts",
"type": "stat"
},
{
@@ -462,12 +514,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 5,
"x": 0,
- "y": 5
+ "y": 5,
+ "w": 6,
+ "h": 4
},
- "id": 243,
+ "id": 386,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -634,12 +686,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 5,
- "x": 5,
- "y": 5
+ "x": 6,
+ "y": 5,
+ "w": 6,
+ "h": 4
},
- "id": 247,
+ "id": 387,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -782,6 +834,172 @@
"title": "Database Resources",
"type": "stat"
},
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Total number of data file size.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 5,
+ "w": 4,
+ "h": 4
+ },
+ "id": 388,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "select SUM(disk_size) from information_schema.region_statistics;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Total Storage Size",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "sishort"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 5,
+ "w": 4,
+ "h": 4
+ },
+ "id": 389,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "select SUM(region_rows) from information_schema.region_statistics;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Total Rows",
+ "type": "stat"
+ },
{
"datasource": {
"type": "mysql",
@@ -807,12 +1025,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 5,
- "x": 10,
- "y": 5
+ "x": 20,
+ "y": 5,
+ "w": 4,
+ "h": 4
},
- "id": 278,
+ "id": 390,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -926,15 +1144,449 @@
"title": "Data Size",
"type": "stat"
},
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 9,
+ "w": 12,
+ "h": 6
+ },
+ "id": 538,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "ingestion",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Ingestion Rate Trend",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 9,
+ "w": 12,
+ "h": 6
+ },
+ "id": 539,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "queries",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Query Rate Trend",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 15,
+ "w": 12,
+ "h": 6
+ },
+ "id": 392,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "HTTP Request P99 and Avg",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Tail and average latency for gRPC requests served by frontends.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 15,
+ "w": 12,
+ "h": 6
+ },
+ "id": 557,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "gRPC Request P99 and Avg",
+ "type": "timeseries"
+ },
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 9
+ "y": 21
},
- "id": 275,
+ "id": 419,
"panels": [],
"title": "Ingestion",
"type": "row"
@@ -1001,12 +1653,12 @@
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 24,
"x": 0,
- "y": 10
+ "y": 22,
+ "w": 8,
+ "h": 6
},
- "id": 193,
+ "id": 413,
"options": {
"legend": {
"calcs": [
@@ -1046,7 +1698,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
+ "description": "Rows, samples, or documents ingested by primary observability and table-ingestion protocols.",
"fieldConfig": {
"defaults": {
"color": {
@@ -1103,12 +1755,12 @@
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 16
+ "x": 8,
+ "y": 22,
+ "w": 8,
+ "h": 6
},
- "id": 277,
+ "id": 414,
"options": {
"legend": {
"calcs": [
@@ -1132,12 +1784,12 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))",
"hide": false,
"instant": false,
- "legendFormat": "http-logs",
+ "legendFormat": "table-operator",
"range": true,
- "refId": "http_logs"
+ "refId": "A"
},
{
"datasource": {
@@ -1145,15 +1797,4695 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))",
+ "expr": "sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~\"$frontend\"}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "prometheus-remote-write",
"range": true,
- "refId": "prometheus-remote-write"
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "loki-logs",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "elasticsearch-docs",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_metrics_rows{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-metrics",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_logs_rows{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-logs",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_traces_rows{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-traces",
+ "range": true,
+ "refId": "H"
}
],
- "title": "Ingestion Rate by Type",
+ "title": "Ingestion Rate by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 22,
+ "w": 8,
+ "h": 6
+ },
+ "id": 415,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "prometheus-write",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "loki-logs",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "elasticsearch",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-metrics",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-logs",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-traces",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "prom-write-avg",
+ "range": true,
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs-avg",
+ "range": true,
+ "refId": "I"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "loki-logs-avg",
+ "range": true,
+ "refId": "J"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "elasticsearch-avg",
+ "range": true,
+ "refId": "K"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-metrics-avg",
+ "range": true,
+ "refId": "L"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-logs-avg",
+ "range": true,
+ "refId": "M"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-traces-avg",
+ "range": true,
+ "refId": "N"
+ }
+ ],
+ "title": "Ingestion Latency by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "Bps"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 28,
+ "w": 8,
+ "h": 6
+ },
+ "id": 416,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "rows",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "bytes",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Bulk Insert Message Rows and Size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Remote-write pending-row flush operations, flushed rows, and p99 flush latency.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 28,
+ "w": 8,
+ "h": 6
+ },
+ "id": 417,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_prom_store_flush_total{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-ops",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_prom_store_flush_rows_sum{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-rows",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-p99",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Prom Store Flush Pipeline",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "OTLP trace ingestion failures reported by frontends.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 28,
+ "w": 8,
+ "h": 6
+ },
+ "id": 418,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trace-failures",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OTLP Trace Failures",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 403,
+ "panels": [],
+ "title": "Health",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 35,
+ "w": 8,
+ "h": 6
+ },
+ "id": 394,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (protocol) (rate(greptime_servers_error{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "server-{{protocol}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (code) (rate(greptime_servers_auth_failure_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "auth-{{code}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\",code!~\"2..\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-{{method}}-{{path}}-{{code}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\",code!~\"0|OK\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-{{path}}-{{code}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Protocol Error Rates",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 35,
+ "w": 8,
+ "h": 6
+ },
+ "id": 395,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_request_memory_rejected_total{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "request-memory",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "query-memory",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Frontend and Query Rejections",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 35,
+ "w": 8,
+ "h": 6
+ },
+ "id": 396,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "region-request-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Datanode Write Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 41,
+ "w": 8,
+ "h": 6
+ },
+ "id": 397,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_pending_rows_flush_failures{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-failures",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "dropped-rows",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Buffered Ingestion Loss",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 41,
+ "w": 8,
+ "h": 6
+ },
+ "id": 398,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_write_reject_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "write-reject",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_write_stall_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "write-stall",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_flush_failure_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-failure",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_compaction_failure_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-failure",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Mito Backpressure and Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Datanode scan and compaction memory rejection/exhaustion counters.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 41,
+ "w": 8,
+ "h": 6
+ },
+ "id": 399,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_scan_requests_rejected_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "scan-rejected",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "scan-exhausted",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-rejected",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Scan and Compaction Memory Rejects",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Object-store errors by scheme, operation, and error, excluding NotFound noise.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 47,
+ "w": 8,
+ "h": 6
+ },
+ "id": 400,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\",error!=\"NotFound\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{scheme}}-{{operation}}-{{error}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OpenDAL Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region migration and reconciliation failures in metasrv.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 47,
+ "w": 8,
+ "h": 6
+ },
+ "id": 401,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "migration-fail",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "reconciliation-error",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Metasrv Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Derived-data and alerting pipeline failures.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 47,
+ "w": 8,
+ "h": 6
+ },
+ "id": 402,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (code) (rate(greptime_flow_errors[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flow-{{code}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trigger-evaluate",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trigger-send-alert",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trigger-save-alert",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Flow and Trigger Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito garbage-collection errors and skipped/orphaned files on datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 53,
+ "w": 8,
+ "h": 6
+ },
+ "id": 550,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_errors_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "gc-errors",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_orphaned_index_files{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "orphaned-index-files",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "skipped-unparsable",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Mito GC Failures",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 59
+ },
+ "id": 411,
+ "panels": [],
+ "title": "Capacity",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 60,
+ "w": 8,
+ "h": 6
+ },
+ "id": 404,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_runtime_threads_alive)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "alive-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_runtime_threads_idle)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "idle-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Runtime Threads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Frontend request memory usage divided by configured request memory limit.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 60,
+ "w": 8,
+ "h": 6
+ },
+ "id": 405,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~\"$frontend\"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request Memory Utilization",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 60,
+ "w": 8,
+ "h": 6
+ },
+ "id": 406,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Query Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Datanode scan memory usage and compaction memory utilization.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 66,
+ "w": 8,
+ "h": 6
+ },
+ "id": 407,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "scan-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-used-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-limit-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Scan and Compaction Memory",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 66,
+ "w": 8,
+ "h": 6
+ },
+ "id": 408,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "buffer-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "stalling-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Write Buffer and Active Stalling",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 66,
+ "w": 8,
+ "h": 6
+ },
+ "id": 409,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "rows-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "batches-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "workers-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Prom Store Backlog",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current in-flight flush and compaction tasks on datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 72,
+ "w": 8,
+ "h": 6
+ },
+ "id": 410,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Inflight Flush and Compaction",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 78
+ },
+ "id": 412,
+ "title": "Resources",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 79
+ },
+ "id": 465,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Frontend CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 79
+ },
+ "id": 463,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Datanode CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 79
+ },
+ "id": 467,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$metasrv\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Metasrv CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 87
+ },
+ "id": 464,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Frontend Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 87
+ },
+ "id": 462,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Datanode Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 87
+ },
+ "id": 466,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$metasrv\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Metasrv Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 95
+ },
+ "id": 469,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$flownode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flownode CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 95
+ },
+ "id": 468,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$flownode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flownode Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 103
+ },
+ "id": 427,
+ "panels": [],
+ "title": "Queries",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Query API call rates by protocol, collected from frontends.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 104,
+ "w": 8,
+ "h": 6
+ },
+ "id": 420,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_sql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-sql",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-query",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Query Rate by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "p95, p99, and average query latency by main frontend protocol.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 104,
+ "w": 8,
+ "h": 6
+ },
+ "id": 421,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-p95",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-p99",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql-p95",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~\"$frontend\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql-p99",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-avg",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-avg",
+ "range": true,
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql-avg",
+ "range": true,
+ "refId": "I"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-query-avg",
+ "range": true,
+ "refId": "J"
+ }
+ ],
+ "title": "Query Latency by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 104,
+ "w": 8,
+ "h": 6
+ },
+ "id": 422,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p95-{{stage}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99-{{stage}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Query Stage Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 110,
+ "w": 8,
+ "h": 6
+ },
+ "id": 423,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_query_merge_scan_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "regions-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "errors-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Merge Scan Fan-out and Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 110,
+ "w": 8,
+ "h": 6
+ },
+ "id": 424,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "pushdown-fallback-errors",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Pushdown Fallback Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 110,
+ "w": 8,
+ "h": 6
+ },
+ "id": 425,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_promql_series_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "PromQL Series Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 116,
+ "w": 8,
+ "h": 6
+ },
+ "id": 426,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-connections-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~\"$frontend\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-connections-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-prepared-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-prepared-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Connections and Prepared Statements",
"type": "timeseries"
},
{
@@ -1162,1092 +6494,9 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 22
+ "y": 122
},
- "id": 276,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "reqps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 246
- },
- "id": 255,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "mysql",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "pg",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~\"$frontend\"}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "promql",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "Total Query Rate",
- "type": "timeseries"
- }
- ],
- "title": "Queries",
- "type": "row"
- },
- {
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 23
- },
- "id": 274,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 247
- },
- "id": 256,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{instance=~\"$datanode\"}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$datanode\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Datanode Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 247
- },
- "id": 262,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$datanode\"}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$datanode\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Datanode CPU Usage per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 257
- },
- "id": 266,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{instance=~\"$frontend\"}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$frontend\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Frontend Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 257
- },
- "id": 268,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$frontend\"}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$frontend\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Frontend CPU Usage per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 267
- },
- "id": 269,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{instance=~\"$metasrv\"}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$metasrv\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Metasrv Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 267
- },
- "id": 271,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$metasrv\"}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$metasrv\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Metasrv CPU Usage per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 277
- },
- "id": 272,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{instance=~\"$flownode\"}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{instance=~\"$flownode\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Flownode Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 277
- },
- "id": 273,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{instance=~\"$flownode\"}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{instance=~\"$flownode\"})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Flownode CPU Usage per Instance",
- "type": "timeseries"
- }
- ],
- "title": "Resources",
- "type": "row"
- },
- {
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 280,
+ "id": 428,
"panels": [
{
"datasource": {
@@ -2312,11 +6561,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 288
+ "y": 123
},
- "id": 281,
+ "id": 470,
"options": {
"legend": {
"calcs": [
@@ -2355,7 +6604,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "HTTP P99 per Instance.",
+ "description": "HTTP P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2413,11 +6662,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 288
+ "w": 8,
+ "x": 8,
+ "y": 123
},
- "id": 282,
+ "id": 471,
"options": {
"legend": {
"calcs": [
@@ -2446,9 +6695,22 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~\"$frontend\",path!~\"/health|/metrics\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "HTTP P99 per Instance",
+ "title": "HTTP P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -2514,11 +6776,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 296
+ "w": 8,
+ "x": 16,
+ "y": 123
},
- "id": 283,
+ "id": 472,
"options": {
"legend": {
"calcs": [
@@ -2557,7 +6819,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "gRPC P99 per Instance.",
+ "description": "gRPC P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2615,11 +6877,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 296
+ "w": 8,
+ "x": 0,
+ "y": 131
},
- "id": 284,
+ "id": 473,
"options": {
"legend": {
"calcs": [],
@@ -2646,9 +6908,22 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "gRPC P99 per Instance",
+ "title": "gRPC P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -2714,11 +6989,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 304
+ "w": 8,
+ "x": 8,
+ "y": 131
},
- "id": 285,
+ "id": 474,
"options": {
"legend": {
"calcs": [
@@ -2757,7 +7032,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "MySQL P99 per Instance.",
+ "description": "MySQL P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2815,11 +7090,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 304
+ "w": 8,
+ "x": 16,
+ "y": 131
},
- "id": 286,
+ "id": 475,
"options": {
"legend": {
"calcs": [
@@ -2849,9 +7124,22 @@
"legendFormat": "[{{ instance }}]-[{{ pod }}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "MySQL P99 per Instance",
+ "title": "MySQL P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -2917,11 +7205,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 312
+ "y": 139
},
- "id": 287,
+ "id": 476,
"options": {
"legend": {
"calcs": [
@@ -2960,7 +7248,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "PostgreSQL P99 per Instance.",
+ "description": "PostgreSQL P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3018,11 +7306,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 312
+ "w": 8,
+ "x": 8,
+ "y": 139
},
- "id": 288,
+ "id": 477,
"options": {
"legend": {
"calcs": [],
@@ -3049,9 +7337,22 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "PostgreSQL P99 per Instance",
+ "title": "PostgreSQL P99 and Avg per Instance",
"type": "timeseries"
}
],
@@ -3064,111 +7365,10 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 25
+ "y": 123
},
- "id": 289,
+ "id": 429,
"panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ingestion rate by row as in each frontend",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 289
- },
- "id": 292,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~\"$frontend\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Ingest Rows per Instance",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -3232,11 +7432,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 295
+ "y": 124
},
- "id": 290,
+ "id": 479,
"options": {
"legend": {
"calcs": [
@@ -3275,7 +7475,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Region Call P99 per Instance.",
+ "description": "Region Call P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3333,11 +7533,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 295
+ "w": 8,
+ "x": 8,
+ "y": 124
},
- "id": 291,
+ "id": 480,
"options": {
"legend": {
"calcs": [
@@ -3366,9 +7566,22 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~\"$frontend\"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Region Call P99 per Instance",
+ "title": "Region Call P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -3430,43 +7643,15 @@
},
"unit": "s"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[minipc-1:4000]-[]-[encode]-AVG",
- "[minipc-1:4000]-[]-[partition]-AVG",
- "[minipc-1:4000]-[]-[decode_request]-AVG",
- "[minipc-1:4000]-[]-[datanode_handle]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 303
+ "w": 8,
+ "x": 16,
+ "y": 124
},
- "id": 336,
+ "id": 481,
"options": {
"legend": {
"calcs": [
@@ -3510,22 +7695,662 @@
"refId": "B"
}
],
- "title": "Frontend Handle Bulk Insert Elapsed Time ",
+ "title": "Frontend Handle Bulk Insert Elapsed Time",
"type": "timeseries"
}
],
"title": "Frontend to Datanode",
"type": "row"
},
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 124
+ },
+ "id": 434,
+ "panels": [],
+ "title": "Datanode",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Datanode region request failures and failed inserts by instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 125,
+ "w": 8,
+ "h": 6
+ },
+ "id": 430,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "request-fail-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Request Failures and Failed Inserts",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito write rejects, write stall events, and active write stalling by datanode.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 125,
+ "w": 8,
+ "h": 6
+ },
+ "id": 431,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "reject-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "stall-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "active-stalling-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Write Rejects and Stalls",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito flush and compaction failure rates by datanode.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 125,
+ "w": 8,
+ "h": 6
+ },
+ "id": 432,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flush and Compaction Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 131,
+ "w": 8,
+ "h": 6
+ },
+ "id": 433,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_runs_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "runs",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_errors_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "errors",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_files_deleted_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "files-deleted",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_orphaned_index_files{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "orphaned-index-files",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "skipped-unparsable",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Mito GC Health",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "P99 and average Mito garbage-collection duration by stage.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 131,
+ "w": 8,
+ "h": 6
+ },
+ "id": 551,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Mito GC Duration",
+ "type": "timeseries"
+ },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 26
+ "y": 137
},
- "id": 293,
+ "id": 435,
"panels": [
{
"datasource": {
@@ -3590,11 +8415,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 193
+ "y": 138
},
- "id": 294,
+ "id": 482,
"options": {
"legend": {
"calcs": [
@@ -3633,7 +8458,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Request P99 per Instance.",
+ "description": "Request P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3691,11 +8516,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 193
+ "w": 8,
+ "x": 8,
+ "y": 138
},
- "id": 295,
+ "id": 483,
"options": {
"legend": {
"calcs": [
@@ -3724,9 +8549,150 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Request P99 per Instance",
+ "title": "Request P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 138
+ },
+ "id": 556,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Request Wait P99 and Avg per Worker",
"type": "timeseries"
},
{
@@ -3792,11 +8758,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 225
+ "y": 146
},
- "id": 296,
+ "id": 484,
"options": {
"legend": {
"calcs": [
@@ -3893,11 +8859,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 225
+ "w": 8,
+ "x": 8,
+ "y": 146
},
- "id": 297,
+ "id": 485,
"options": {
"legend": {
"calcs": [
@@ -3931,207 +8897,6 @@
"title": "Write Rows per Instance",
"type": "timeseries"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Flush QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 233
- },
- "id": 298,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Flush OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stall per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 233
- },
- "id": 299,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{instance=~\"$datanode\"})",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stall per Instance",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -4195,11 +8960,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 241
+ "w": 8,
+ "x": 16,
+ "y": 146
},
- "id": 300,
+ "id": 488,
"options": {
"legend": {
"calcs": [
@@ -4238,133 +9003,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Read Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 241
- },
- "id": 301,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stage P99 per Instance.",
+ "description": "Read Stage P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -4422,11 +9061,125 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 249
+ "y": 154
},
- "id": 302,
+ "id": 489,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Read Stage P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stage P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 154
+ },
+ "id": 490,
"options": {
"legend": {
"calcs": [
@@ -4457,208 +9210,6 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
"range": true,
"refId": "A"
- }
- ],
- "title": "Write Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 249
- },
- "id": 303,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction latency by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 257
- },
- "id": 304,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
- "range": true,
- "refId": "A"
},
{
"datasource": {
@@ -4666,7 +9217,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
@@ -4674,235 +9225,7 @@
"refId": "B"
}
],
- "title": "Compaction Elapsed Time per Instance by Stage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 257
- },
- "id": 305,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 265
- },
- "id": 306,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "WAL write size",
+ "title": "Write Stage P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -4964,40 +9287,15 @@
},
"unit": "decbytes"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 265
+ "w": 8,
+ "x": 16,
+ "y": 154
},
- "id": 307,
+ "id": 495,
"options": {
"legend": {
"calcs": [],
@@ -5029,524 +9327,6 @@
"title": "Cached Bytes per Instance",
"type": "timeseries"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing compaction task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 273
- },
- "id": 308,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_compaction_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Compaction",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Raft engine (local disk) log store sync latency, p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 273
- },
- "id": 310,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "WAL sync duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead log operations latency at p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 281
- },
- "id": 311,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Log Store op duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing flush task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 281
- },
- "id": 312,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_flush_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Flush",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 289
- },
- "id": 335,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-input",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-output",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Input/Output Bytes",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -5606,44 +9386,15 @@
},
"unit": "s"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[bulk_extend]-AVG",
- "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
- "[minipc-1:4000]-[]-[write_bulk]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 289
+ "w": 8,
+ "x": 0,
+ "y": 162
},
- "id": 337,
+ "id": 501,
"options": {
"legend": {
"calcs": [
@@ -5695,7 +9446,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Compaction oinput output bytes",
+ "description": "Active series and field-builder counts per memtable by instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -5753,11 +9504,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 297
+ "w": 8,
+ "x": 8,
+ "y": 162
},
- "id": 348,
+ "id": 502,
"options": {
"legend": {
"calcs": [
@@ -5867,11 +9618,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 297
+ "w": 8,
+ "x": 16,
+ "y": 162
},
- "id": 338,
+ "id": 503,
"options": {
"legend": {
"calcs": [
@@ -5984,11 +9735,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 305
+ "y": 170
},
- "id": 349,
+ "id": 504,
"options": {
"legend": {
"calcs": [
@@ -6022,7 +9773,7 @@
"type": "timeseries"
}
],
- "title": "Mito Engine",
+ "title": "Storage",
"type": "row"
},
{
@@ -6031,16 +9782,16 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 27
+ "y": 138
},
- "id": 313,
+ "id": 555,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "QPS per Instance.",
+ "description": "Flush QPS per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -6053,6 +9804,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -6096,12 +9848,12 @@
"overrides": []
},
"gridPos": {
- "h": 10,
- "w": 24,
+ "h": 8,
+ "w": 8,
"x": 0,
- "y": 459
+ "y": 139
},
- "id": 314,
+ "id": 486,
"options": {
"legend": {
"calcs": [
@@ -6117,7 +9869,7 @@
"sort": "desc"
}
},
- "pluginVersion": "12.0.0",
+ "pluginVersion": "11.6.0",
"targets": [
{
"datasource": {
@@ -6125,14 +9877,14 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~\"$datanode\"}[$__rate_interval]))",
"instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
"range": true,
"refId": "A"
}
],
- "title": "QPS per Instance",
+ "title": "Flush OPS per Instance",
"type": "timeseries"
},
{
@@ -6140,7 +9892,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Read QPS per Instance.",
+ "description": "Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs.",
"fieldConfig": {
"defaults": {
"color": {
@@ -6153,6 +9905,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -6191,823 +9944,21 @@
}
]
},
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 469
- },
- "id": 315,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read QPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
"unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 469
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 139
},
- "id": 316,
+ "id": 540,
"options": {
"legend": {
"calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=~\"read|Reader::read\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 476
- },
- "id": 317,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write QPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 476
- },
- "id": 318,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "List QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 483
- },
- "id": 319,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))",
- "instant": false,
- "interval": "",
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "List QPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "List P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 483
- },
- "id": 320,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))",
- "instant": false,
- "interval": "",
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "List P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Other Requests per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 490
- },
- "id": 321,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\",operation!~\"read|write|list|stat\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Other Requests per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Other Request P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 3,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 490
- },
- "id": 322,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|write|list|Writer::write|Writer::close|Reader::read\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Other Request P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Total traffic as in bytes by instance and operation",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 497
- },
- "id": 323,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
+ "mean"
],
"displayMode": "table",
"placement": "bottom",
@@ -7019,26 +9970,49 @@
"sort": "none"
}
},
- "pluginVersion": "12.0.0",
+ "pluginVersion": "11.6.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "disableTextWrap": false,
"editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
+ "expr": "histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "hide": false,
"instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95",
"range": true,
- "refId": "A",
- "useBackend": false
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg",
+ "range": true,
+ "refId": "C"
}
],
- "title": "Opendal traffic",
+ "title": "Flush Elapsed Time",
"type": "timeseries"
},
{
@@ -7046,7 +10020,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "OpenDAL error counts per Instance.",
+ "description": "Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7059,6 +10033,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -7096,17 +10071,247 @@
"value": 80
}
]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*-files$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
}
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 139
+ },
+ "id": 541,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-bytes",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-files",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flush Throughput",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing flush task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 497
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 147
},
- "id": 334,
+ "id": 499,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_flush_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Flush",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 147
+ },
+ "id": 491,
"options": {
"legend": {
"calcs": [
@@ -7122,7 +10327,7 @@
"sort": "desc"
}
},
- "pluginVersion": "12.0.0",
+ "pluginVersion": "11.6.0",
"targets": [
{
"datasource": {
@@ -7130,18 +10335,461 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\", error!=\"NotFound\"}[$__rate_interval]))",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
"instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]",
+ "legendFormat": "[{{ instance }}]-[{{pod}}]",
"range": true,
"refId": "A"
}
],
- "title": "OpenDAL errors per Instance",
+ "title": "Compaction OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing compaction task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 147
+ },
+ "id": 496,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_compaction_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Compaction",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 155
+ },
+ "id": 493,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction latency by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 155
+ },
+ "id": 492,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~\"$datanode\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~\"$datanode\"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Elapsed Time per Instance by Stage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 155
+ },
+ "id": 500,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-input",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-output",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Input/Output Bytes",
"type": "timeseries"
}
],
- "title": "OpenDAL",
+ "title": "Flush and Compaction",
"type": "row"
},
{
@@ -7150,16 +10798,16 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 28
+ "y": 139
},
- "id": 351,
+ "id": 549,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Triggered region flush total",
+ "description": "Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7211,21 +10859,23 @@
}
]
},
- "unit": "none"
+ "unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 195
+ "y": 164,
+ "w": 8,
+ "h": 6
},
- "id": 350,
+ "id": 542,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7243,14 +10893,28 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "meta_triggered_region_flush_total",
+ "expr": "histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
+ "legendFormat": "{{type}}-p95",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-p99",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Triggered region flush total",
+ "title": "Index Apply Elapsed Time",
"type": "timeseries"
},
{
@@ -7258,7 +10922,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Triggered region checkpoint total",
+ "description": "Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7310,21 +10974,23 @@
}
]
},
- "unit": "none"
+ "unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 195
+ "x": 8,
+ "y": 164,
+ "w": 8,
+ "h": 6
},
- "id": 352,
+ "id": 543,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7342,14 +11008,28 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "meta_triggered_region_checkpoint_total",
+ "expr": "histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
+ "legendFormat": "{{type}}-{{stage}}-p95",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-{{stage}}-p99",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Triggered region checkpoint total",
+ "title": "Index Create Elapsed Time",
"type": "timeseries"
},
{
@@ -7357,7 +11037,135 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Topic estimated max replay size",
+ "description": "Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*-bytes$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "Bps"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 164,
+ "w": 8,
+ "h": 6
+ },
+ "id": 544,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-rows",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-bytes",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Index Create Rows and Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7414,16 +11222,18 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 227
+ "y": 170,
+ "w": 8,
+ "h": 6
},
- "id": 353,
+ "id": 545,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7441,14 +11251,28 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "meta_topic_estimated_replay_size",
+ "expr": "greptime_index_apply_memory_usage",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
+ "legendFormat": "apply",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (greptime_index_create_memory_usage)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "create-{{type}}",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Topic estimated replay size",
+ "title": "Index Memory Usage",
"type": "timeseries"
},
{
@@ -7456,7 +11280,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Kafka logstore's bytes traffic",
+ "description": "Index read and write byte rates by operation and file type for puffin and intermediate files.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7508,21 +11332,23 @@
}
]
},
- "unit": "bytes"
+ "unit": "Bps"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 227
+ "x": 8,
+ "y": 170,
+ "w": 8,
+ "h": 6
},
- "id": 354,
+ "id": 546,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7540,18 +11366,249 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
+ "expr": "sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{logstore}}",
+ "legendFormat": "{{type}}-{{file_type}}",
"range": true,
"refId": "A"
}
],
- "title": "Kafka logstore's bytes traffic",
+ "title": "Index IO Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Index IO operation rates by operation and file type, including read, write, seek, and flush operations.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 170,
+ "w": 8,
+ "h": 6
+ },
+ "id": 547,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-{{file_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Index IO Operations",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Index-related cache hits, misses, and evictions from Mito caches.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 176,
+ "w": 8,
+ "h": 6
+ },
+ "id": 548,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "hit-{{type}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "miss-{{type}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "evict-{{type}}-{{cause}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Index Cache",
"type": "timeseries"
}
],
- "title": "Remote WAL",
+ "title": "Index",
"type": "row"
},
{
@@ -7560,10 +11617,292 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 29
+ "y": 140
},
- "id": 324,
+ "id": 451,
"panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 165,
+ "w": 8,
+ "h": 8
+ },
+ "id": 436,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(greptime_meta_inactive_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "inactive-regions",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(greptime_lease_expired_region{instance=~\"$datanode\"})",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "lease-expired",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Inactive and Lease-expired Regions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 165,
+ "w": 8,
+ "h": 8
+ },
+ "id": 437,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "meta-recv-rate",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(greptime_meta_heartbeat_connection_num)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "meta-connections",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_heartbeat_send_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "frontend-send",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_heartbeat_recv_count{instance=~\"$frontend\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "frontend-recv",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_datanode_heartbeat_send_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "datanode-send",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_datanode_heartbeat_recv_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "datanode-recv",
+ "range": true,
+ "refId": "F"
+ }
+ ],
+ "title": "Heartbeat Health",
+ "type": "timeseries"
+ },
{
"datasource": {
"type": "prometheus",
@@ -7602,12 +11941,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 196
+ "x": 16,
+ "y": 165,
+ "w": 8,
+ "h": 8
},
- "id": 325,
+ "id": 525,
"options": {
"colWidth": 0.9,
"legend": {
@@ -7716,12 +12055,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 196
+ "x": 0,
+ "y": 173,
+ "w": 8,
+ "h": 8
},
- "id": 326,
+ "id": 526,
"options": {
"legend": {
"calcs": [],
@@ -7743,7 +12082,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "greptime_meta_region_migration_error",
+ "expr": "rate(greptime_meta_region_migration_error[$__rate_interval])",
"instant": false,
"legendFormat": "{{pod}}-{{state}}-{{error_type}}",
"range": true,
@@ -7815,12 +12154,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 228
+ "x": 8,
+ "y": 173,
+ "w": 8,
+ "h": 8
},
- "id": 327,
+ "id": 527,
"options": {
"legend": {
"calcs": [],
@@ -7914,12 +12253,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 228
+ "x": 16,
+ "y": 173,
+ "w": 8,
+ "h": 8
},
- "id": 339,
+ "id": 528,
"options": {
"legend": {
"calcs": [],
@@ -8013,12 +12352,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 236
+ "y": 181,
+ "w": 8,
+ "h": 8
},
- "id": 340,
+ "id": 529,
"options": {
"legend": {
"calcs": [],
@@ -8049,6 +12388,19 @@
"range": true,
"refId": "A",
"useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
"title": "SQL Execution Latency (RDS)",
@@ -8116,12 +12468,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 236
+ "x": 8,
+ "y": 181,
+ "w": 8,
+ "h": 8
},
- "id": 341,
+ "id": 530,
"options": {
"legend": {
"calcs": [],
@@ -8152,6 +12504,19 @@
"range": true,
"refId": "A",
"useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{pod}}]-[{{name}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
"title": "Handler Execution Latency",
@@ -8219,12 +12584,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 244
+ "x": 16,
+ "y": 181,
+ "w": 8,
+ "h": 8
},
- "id": 342,
+ "id": 531,
"options": {
"legend": {
"calcs": [],
@@ -8247,7 +12612,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
+ "expr": "histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
@@ -8265,7 +12630,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "Rate of heartbeats received by metasrv from datanodes and frontends.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8322,12 +12687,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 244
+ "x": 0,
+ "y": 189,
+ "w": 8,
+ "h": 8
},
- "id": 345,
+ "id": 532,
"options": {
"legend": {
"calcs": [],
@@ -8368,7 +12733,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "p99 and average latency of metasrv key-value store operations by op and target.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8425,12 +12790,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 252
+ "x": 8,
+ "y": 189,
+ "w": 8,
+ "h": 8
},
- "id": 343,
+ "id": 533,
"options": {
"legend": {
"calcs": [],
@@ -8453,7 +12818,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
+ "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
@@ -8461,6 +12826,19 @@
"range": true,
"refId": "A",
"useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{pod}}]-[{{op}}]-[{{target}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
"title": "Meta KV Ops Latency",
@@ -8471,7 +12849,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "Rate of metasrv key-value store operations by op.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8528,12 +12906,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 252
+ "x": 16,
+ "y": 189,
+ "w": 8,
+ "h": 8
},
- "id": 346,
+ "id": 534,
"options": {
"legend": {
"calcs": [],
@@ -8574,7 +12952,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8631,12 +13009,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 260
+ "y": 197,
+ "w": 8,
+ "h": 8
},
- "id": 347,
+ "id": 535,
"options": {
"legend": {
"calcs": [],
@@ -8659,7 +13037,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
@@ -8675,7 +13053,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8692,7 +13070,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8709,7 +13087,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8726,7 +13104,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8743,7 +13121,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8814,17 +13192,17 @@
}
]
},
- "unit": "s"
+ "unit": "ops"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 260
+ "x": 8,
+ "y": 197,
+ "w": 8,
+ "h": 8
},
- "id": 355,
+ "id": 536,
"options": {
"legend": {
"calcs": [],
@@ -8846,7 +13224,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "greptime_meta_reconciliation_stats",
+ "expr": "rate(greptime_meta_reconciliation_stats[$__rate_interval])",
"hide": false,
"instant": false,
"legendFormat": "{{pod}}-{{table_type}}-{{type}}",
@@ -8919,12 +13297,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 268
+ "x": 16,
+ "y": 197,
+ "w": 8,
+ "h": 8
},
- "id": 356,
+ "id": 537,
"options": {
"legend": {
"calcs": [],
@@ -8946,7 +13324,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)",
+ "expr": "histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "{{procedure_name}}-{{step}}-P90",
@@ -8967,9 +13345,3174 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 30
+ "y": 141
},
- "id": 328,
+ "id": 553,
+ "panels": [
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto",
+ "wrapText": false
+ },
+ "inspect": false
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "disk_size"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 166,
+ "w": 24,
+ "h": 18
+ },
+ "id": 438,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "enablePagination": true,
+ "fields": [
+ "region_rows",
+ "disk_size"
+ ],
+ "reducer": [
+ "sum"
+ ],
+ "show": true
+ },
+ "showHeader": true,
+ "sortBy": []
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Hotspot Regions",
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write load of each datanode over time.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 184,
+ "w": 8,
+ "h": 11
+ },
+ "id": 439,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "greptime_datanode_history_load",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Distribution of write load across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "min": 0,
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 184,
+ "w": 8,
+ "h": 11
+ },
+ "id": 440,
+ "options": {
+ "displayLabels": [
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "pieType": "donut",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "exemplar": false,
+ "expr": "greptime_datanode_history_load",
+ "format": "time_series",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": true,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": false,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write) Distribution",
+ "type": "piechart"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Distribution of leader regions and data size across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 184,
+ "w": 8,
+ "h": 11
+ },
+ "id": 441,
+ "options": {
+ "displayLabels": [
+ "value"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "percent"
+ ]
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "/^data_size$/",
+ "values": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Datanode Data Distribution",
+ "type": "piechart"
+ }
+ ],
+ "title": "Hotspot",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 142
+ },
+ "id": 554,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer action count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 167,
+ "w": 8,
+ "h": 8
+ },
+ "id": 442,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 167,
+ "w": 8,
+ "h": 8
+ },
+ "id": 443,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 167,
+ "w": 8,
+ "h": 8
+ },
+ "id": 444,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_datanodes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanodes",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer region count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 175,
+ "w": 8,
+ "h": 8
+ },
+ "id": 445,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Regions",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode stability statistics by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*count$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*ratio$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percentunit"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 175,
+ "w": 8,
+ "h": 8
+ },
+ "id": 446,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "builder",
+ "expr": "sum by (state) (greptime_region_balancer_datanode_stability)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanode Stability",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition action count by result",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": -1,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 175,
+ "w": 8,
+ "h": 8
+ },
+ "id": 447,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 183,
+ "w": 8,
+ "h": 8
+ },
+ "id": 448,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition sampling elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 183,
+ "w": 8,
+ "h": 8
+ },
+ "id": 449,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Sampling P99",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition executor elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 183,
+ "w": 8,
+ "h": 8
+ },
+ "id": 450,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Executor P99",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Autopilot",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 143
+ },
+ "id": 452,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 168
+ },
+ "id": 505,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 178
+ },
+ "id": 506,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 178
+ },
+ "id": 507,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\",operation=~\"read|Reader::read\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"read|Reader::read\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Read P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 178
+ },
+ "id": 508,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 185
+ },
+ "id": 509,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Write P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "List QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 185
+ },
+ "id": 510,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "List QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "List P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 185
+ },
+ "id": 511,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])))",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation=\"list\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "List P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Other Requests per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 192
+ },
+ "id": 512,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Other Requests per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Other Request P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 3,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 192
+ },
+ "id": 513,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~\"$datanode\", operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Other Request P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total traffic as in bytes by instance and operation",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 192
+ },
+ "id": 514,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~\"$datanode\"}[$__rate_interval]))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Opendal traffic",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "OpenDAL error counts per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 199
+ },
+ "id": 515,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~\"$datanode\", error!=\"NotFound\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OpenDAL errors per Instance",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Object Store",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 144
+ },
+ "id": 552,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 169
+ },
+ "id": 494,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "WAL write size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Raft engine (local disk) log store sync latency, p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 169
+ },
+ "id": 497,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "WAL sync duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead log operations latency at p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 169
+ },
+ "id": 498,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Log Store op duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region flush total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 177
+ },
+ "id": 516,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_flush_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region flush total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region checkpoint total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 177
+ },
+ "id": 517,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_checkpoint_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region checkpoint total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Topic estimated max replay size",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 177
+ },
+ "id": 518,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_topic_estimated_replay_size",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Topic estimated replay size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Kafka logstore's bytes traffic",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 185
+ },
+ "id": 519,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{logstore}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Kafka logstore's bytes traffic",
+ "type": "timeseries"
+ }
+ ],
+ "title": "WAL",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 145
+ },
+ "id": 453,
"panels": [
{
"datasource": {
@@ -9032,11 +16575,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 3282
+ "y": 170
},
- "id": 329,
+ "id": 520,
"options": {
"legend": {
"calcs": [],
@@ -9127,11 +16670,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 3282
+ "w": 8,
+ "x": 8,
+ "y": 170
},
- "id": 330,
+ "id": 521,
"options": {
"legend": {
"calcs": [],
@@ -9169,6 +16712,19 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-p99",
"range": true,
"refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "C"
}
],
"title": "Flow Ingest Latency",
@@ -9235,11 +16791,11 @@
},
"gridPos": {
"h": 8,
- "w": 9,
- "x": 0,
- "y": 3290
+ "w": 8,
+ "x": 16,
+ "y": 170
},
- "id": 331,
+ "id": 522,
"options": {
"legend": {
"calcs": [],
@@ -9343,11 +16899,11 @@
},
"gridPos": {
"h": 8,
- "w": 9,
- "x": 9,
- "y": 3290
+ "w": 8,
+ "x": 0,
+ "y": 178
},
- "id": 332,
+ "id": 523,
"options": {
"legend": {
"calcs": [],
@@ -9438,11 +16994,11 @@
},
"gridPos": {
"h": 8,
- "w": 6,
- "x": 18,
- "y": 3290
+ "w": 8,
+ "x": 8,
+ "y": 178
},
- "id": 333,
+ "id": 524,
"options": {
"legend": {
"calcs": [],
@@ -9477,2077 +17033,793 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 31
+ "y": 146
},
- "id": 357,
- "panels": [],
- "title": "Trigger",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "bf9fzta69bhtsa"
- },
- "description": "Total number of triggers currently defined.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
+ "id": 461,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
},
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
+ "description": "Total number of triggers currently defined.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 358,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_trigger_count{}",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Trigger Count",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 40
- },
- "id": 359,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p75",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Trigger Eval Elapsed",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Rate of failed trigger evaluations.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 40
- },
- "id": 360,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Trigger Eval Failure Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed time to send trigger alerts to notification channels.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 48
- },
- "id": 361,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Send Alert Elapsed",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Rate of failures when sending trigger alerts.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 48
- },
- "id": 364,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Send Alert Failure Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed time to persist trigger alert records.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 56
- },
- "id": 363,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Save Alert Elapsed",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Rate of failures when persisting trigger alert records.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 56
- },
- "id": 362,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Save Alert Failure Rate",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 64
- },
- "id": 366,
- "panels": [],
- "title": "Hotspot",
- "type": "row"
- },
- {
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto",
- "wrapText": false
- },
- "inspect": false
- },
- "fieldMinMax": false,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "disk_size"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 18,
- "w": 24,
- "x": 0,
- "y": 65
- },
- "id": 365,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": false,
- "enablePagination": true,
- "fields": [
- "region_rows",
- "disk_size"
- ],
- "reducer": [
- "sum"
- ],
- "show": true
- },
- "showHeader": true,
- "sortBy": []
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "dataset": "information_schema",
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- }
- }
- ],
- "title": "Hotspot Regions",
- "type": "table"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write load of each datanode over time.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "total_region_rows",
- "total_written_bytes_since_open",
- "total_disk_size",
- "total_sst_size",
- "total_sst_num",
- "total_memtable_size",
- "total_manifest_size",
- "total_index_size",
- "cluster_disk_ratio_percent",
- "cluster_sst_ratio_percent",
- "cluster_rows_ratio_percent",
- "max_region_disk_size",
- "max_region_rows",
- "leader_region_count",
- "disk_ratio_percent",
- "data_size"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": []
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 0,
- "y": 83
- },
- "id": 368,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "greptime_datanode_history_load",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "datanode-{{datanode_id}}({{instance}})",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Datanode Load(Write)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Distribution of write load across datanodes.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "min": 0,
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 12,
- "y": 83
- },
- "id": 369,
- "options": {
- "displayLabels": [
- "percent"
- ],
- "legend": {
- "displayMode": "list",
- "placement": "right",
- "showLegend": true
- },
- "pieType": "donut",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "exemplar": false,
- "expr": "greptime_datanode_history_load",
- "format": "time_series",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": true,
- "legendFormat": "datanode-{{datanode_id}}({{instance}})",
- "range": false,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Datanode Load(Write) Distribution",
- "type": "piechart"
- },
- {
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "description": "Distribution of leader regions and data size across datanodes.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "unit": "bytes"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "total_region_rows",
- "total_written_bytes_since_open",
- "total_disk_size",
- "total_sst_size",
- "total_sst_num",
- "total_memtable_size",
- "total_manifest_size",
- "total_index_size",
- "cluster_disk_ratio_percent",
- "cluster_sst_ratio_percent",
- "cluster_rows_ratio_percent",
- "max_region_disk_size",
- "max_region_rows",
- "leader_region_count",
- "disk_ratio_percent",
- "data_size"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
"legend": false,
"tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 0,
- "y": 94
- },
- "id": 367,
- "options": {
- "displayLabels": [
- "value"
- ],
- "legend": {
- "displayMode": "list",
- "placement": "right",
- "showLegend": true,
- "values": [
- "percent"
- ]
- },
- "pieType": "pie",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "/^data_size$/",
- "values": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "dataset": "information_schema",
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
+ "viz": false
},
- "type": "groupBy"
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
}
- ],
- "limit": 50
- }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 171,
+ "w": 8,
+ "h": 6
+ },
+ "id": 454,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_trigger_count{}",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Trigger Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 171,
+ "w": 8,
+ "h": 6
+ },
+ "id": 455,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p75",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Trigger Eval Elapsed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Rate of failed trigger evaluations.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 171,
+ "w": 8,
+ "h": 6
+ },
+ "id": 456,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Trigger Eval Failure Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed time to send trigger alerts to notification channels.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 177,
+ "w": 8,
+ "h": 6
+ },
+ "id": 457,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p75",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Send Alert Elapsed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Rate of failures when sending trigger alerts.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 177,
+ "w": 8,
+ "h": 6
+ },
+ "id": 458,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Send Alert Failure Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed time to persist trigger alert records.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 177,
+ "w": 8,
+ "h": 6
+ },
+ "id": 459,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p75",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Save Alert Elapsed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Rate of failures when persisting trigger alert records.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 183,
+ "w": 8,
+ "h": 6
+ },
+ "id": 460,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Save Alert Failure Rate",
+ "type": "timeseries"
}
],
- "title": "Datanode Data Distribution",
- "type": "piechart"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 105
- },
- "id": 370,
- "panels": [],
- "title": "Autopilot",
+ "title": "Trigger",
"type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer action count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 106
- },
- "id": 371,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{result}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Actions",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer gate stop count by gate and reason",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 106
- },
- "id": 372,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{gate}} / {{reason}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Gate Stops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer datanode count by state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 114
- },
- "id": 373,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (state) (greptime_region_balancer_datanodes)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{state}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Datanodes",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer region count by state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 114
- },
- "id": 374,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (state) (greptime_region_balancer_regions)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{state}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Regions",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer datanode stability statistics by state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": ".*count$"
- },
- "properties": [
- {
- "id": "unit",
- "value": "short"
- }
- ]
- },
- {
- "matcher": {
- "id": "byRegexp",
- "options": ".*ratio$"
- },
- "properties": [
- {
- "id": "unit",
- "value": "percentunit"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 122
- },
- "id": 375,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "builder",
- "expr": "sum by (state) (greptime_region_balancer_datanode_stability)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{state}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Datanode Stability",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition action count by result",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": -1,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineStyle": {
- "fill": "solid"
- },
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 130
- },
- "id": 376,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{result}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Actions",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition gate stop count by gate and reason",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 130
- },
- "id": 377,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{gate}} / {{reason}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Gate Stops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition sampling elapsed time by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 138
- },
- "id": 378,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{stage}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Sampling P99",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition executor elapsed time by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 138
- },
- "id": 379,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{stage}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Executor P99",
- "type": "timeseries"
}
],
"preload": false,
@@ -11686,7 +17958,7 @@
},
"timepicker": {},
"timezone": "",
- "title": "GreptimeDB Copy",
+ "title": "GreptimeDB",
"uid": "dflfbxbwvvchsa",
- "version": 67
-}
\ No newline at end of file
+ "version": 68
+}
diff --git a/grafana/dashboards/metrics/cluster/dashboard.md b/grafana/dashboards/metrics/cluster/dashboard.md
index 2204c6d0a7..fd7ae68fd4 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.md
+++ b/grafana/dashboards/metrics/cluster/dashboard.md
@@ -4,144 +4,154 @@
| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` |
| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- |
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` |
-| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
-| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
+| Total Query Rate | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `stat` | Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. | `prometheus` | `reqps` | `queries` |
+| User-facing Error Rate | `sum(rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))` | `stat` | Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. | `prometheus` | `eps` | `errors` |
+| Recent Restarts | `sum(changes(process_start_time_seconds[$__range]))` | `stat` | Process restarts over the selected time range across GreptimeDB roles. | `prometheus` | `short` | `restarts` |
| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- |
| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- |
+| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
+| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- |
+| Total Ingestion Rate Trend | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. | `prometheus` | `rowsps` | `ingestion` |
+| Total Query Rate Trend | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. | `prometheus` | `reqps` | `queries` |
+| HTTP Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))`
`sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. | `prometheus` | `s` | `http-p99` |
+| gRPC Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Tail and average latency for gRPC requests served by frontends. | `prometheus` | `s` | `grpc-p99` |
# Ingestion
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total ingestion rate.
Here we listed 3 primary protocols:
- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` |
-| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.
Here we listed 3 primary protocols:
- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `http-logs` |
-# Queries
+| Ingestion Rate by Protocol | `sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_metrics_rows{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_logs_rows{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_traces_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Rows, samples, or documents ingested by primary observability and table-ingestion protocols. | `prometheus` | `rowsps` | `table-operator` |
+| Ingestion Latency by Protocol | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. | `prometheus` | `s` | `prometheus-write` |
+| Bulk Insert Message Rows and Size | `sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. | `prometheus` | `rowsps` | `rows` |
+| Prom Store Flush Pipeline | `sum(rate(greptime_prom_store_flush_total{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_prom_store_flush_rows_sum{instance=~"$frontend"}[$__rate_interval]))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Remote-write pending-row flush operations, flushed rows, and p99 flush latency. | `prometheus` | `short` | `flush-ops` |
+| OTLP Trace Failures | `sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | OTLP trace ingestion failures reported by frontends. | `prometheus` | `eps` | `trace-failures` |
+# Health
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.
Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API
Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` |
+| Protocol Error Rates | `sum by (protocol) (rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))`
`sum by (code) (rate(greptime_servers_auth_failure_count{instance=~"$frontend"}[$__rate_interval]))`
`sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics",code!~"2.."}[$__rate_interval]))`
`sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend",code!~"0\|OK"}[$__rate_interval]))` | `timeseries` | User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. | `prometheus` | `eps` | `server-{{protocol}}` |
+| Frontend and Query Rejections | `sum(rate(greptime_servers_request_memory_rejected_total{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))` | `timeseries` | Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. | `prometheus` | `rps` | `request-memory` |
+| Datanode Write Failures | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. | `prometheus` | `eps` | `region-request-[{{instance}}]-[{{pod}}]` |
+| Buffered Ingestion Loss | `sum(rate(greptime_pending_rows_flush_failures{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. | `prometheus` | `eps` | `flush-failures` |
+| Mito Backpressure and Failures | `sum(rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. | `prometheus` | `eps` | `write-reject` |
+| Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` |
+| OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode",error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` |
+| Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`
`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` |
+| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`
`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` |
+| Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` |
+# Capacity
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Runtime Threads | `sum by (instance, pod) (greptime_runtime_threads_alive)`
`sum by (instance, pod) (greptime_runtime_threads_idle)` | `timeseries` | Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. | `prometheus` | `short` | `alive-[{{instance}}]-[{{pod}}]` |
+| Request Memory Utilization | `sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~"$frontend"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~"$frontend"})` | `timeseries` | Frontend request memory usage divided by configured request memory limit. | `prometheus` | `percentunit` | `[{{instance}}]-[{{pod}}]` |
+| Query Memory Usage | `sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)` | `timeseries` | Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]` |
+| Scan and Compaction Memory | `sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~"$datanode"})` | `timeseries` | Datanode scan memory usage and compaction memory utilization. | `prometheus` | `bytes` | `scan-[{{instance}}]-[{{pod}}]` |
+| Write Buffer and Active Stalling | `sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})` | `timeseries` | Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. | `prometheus` | `bytes` | `buffer-[{{instance}}]-[{{pod}}]` |
+| Prom Store Backlog | `sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~"$frontend"})`
`sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~"$frontend"})`
`sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~"$frontend"})` | `timeseries` | Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. | `prometheus` | `short` | `rows-[{{instance}}]-[{{pod}}]` |
+| Inflight Flush and Compaction | `sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~"$datanode"})`
`sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~"$datanode"})` | `timeseries` | Current in-flight flush and compaction tasks on datanodes. | `prometheus` | `short` | `flush-[{{instance}}]-[{{pod}}]` |
# Resources
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$datanode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
-| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$datanode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
-| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$frontend"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$frontend"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` |
-| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$metasrv"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
+| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$datanode"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$datanode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
-| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$flownode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
+| Frontend Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$frontend"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$frontend"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
+| Datanode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$datanode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
+| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$metasrv"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$metasrv"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{instance=~"$flownode"})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
+| Flownode Memory per Instance | `sum(process_resident_memory_bytes{instance=~"$flownode"}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{instance=~"$flownode"})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
+# Queries
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Query Rate by Protocol | `sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Query API call rates by protocol, collected from frontends. | `prometheus` | `reqps` | `mysql` |
+| Query Latency by Protocol | `histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | p95, p99, and average query latency by main frontend protocol. | `prometheus` | `s` | `mysql-p95` |
+| Query Stage Latency | `histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))` | `timeseries` | p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. | `prometheus` | `s` | `p95-{{stage}}` |
+| Merge Scan Fan-out and Errors | `sum by (instance, pod) (greptime_query_merge_scan_regions)`
`sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))` | `timeseries` | Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. | `prometheus` | `short` | `regions-[{{instance}}]-[{{pod}}]` |
+| Pushdown Fallback Errors | `sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))` | `timeseries` | Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. | `prometheus` | `eps` | `pushdown-fallback-errors` |
+| PromQL Series Count | `sum by (instance, pod) (greptime_promql_series_count)` | `timeseries` | Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]` |
+| Connections and Prepared Statements | `sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~"$frontend"})`
`sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~"$frontend"})`
`sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~"$frontend"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. | `prometheus` | `short` | `mysql-connections-[{{instance}}]-[{{pod}}]` |
# Frontend Requests
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` |
-| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
+| HTTP P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])))`
`sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` |
-| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
+| gRPC P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | gRPC P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
-| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
+| MySQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | MySQL P99 and average per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
-| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
+| PostgreSQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | PostgreSQL P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
# Frontend to Datanode
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
-| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
-| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
-# Mito Engine
+| Region Call P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))`
`sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))` | `timeseries` | Region Call P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
+| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
+# Datanode
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Region Request Failures and Failed Inserts | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Datanode region request failures and failed inserts by instance. | `prometheus` | `eps` | `request-fail-[{{instance}}]-[{{pod}}]` |
+| Write Rejects and Stalls | `sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})` | `timeseries` | Mito write rejects, write stall events, and active write stalling by datanode. | `prometheus` | `short` | `reject-[{{instance}}]-[{{pod}}]` |
+| Flush and Compaction Failures | `sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush and compaction failure rates by datanode. | `prometheus` | `eps` | `flush-[{{instance}}]-[{{pod}}]` |
+| Mito GC Health | `sum(rate(greptime_mito_gc_runs_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_files_deleted_total{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. | `prometheus` | `short` | `runs` |
+| Mito GC Duration | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | P99 and average Mito garbage-collection duration by stage. | `prometheus` | `s` | `{{stage}}-p99` |
+# Storage
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
+| Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
+| Request Wait P99 and Avg per Worker | `histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{worker}}]-p95` |
| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{instance=~"$datanode"}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` |
| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
-| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
-| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode", stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` |
-| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
-| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
-| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
-| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
-| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` |
-| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
+| Read Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Read Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
+| Write Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Write Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
| Cached Bytes per Instance | `greptime_mito_cache_bytes{instance=~"$datanode"}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
-| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
-| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
-| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
-| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`
`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`
`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
-| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
+| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Active series and field-builder counts per memtable by instance. | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`
`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-# OpenDAL
+# Flush and Compaction
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
-| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
-| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
-# Remote WAL
+| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
+| Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
+| Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~"$datanode"}[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` |
+| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
+| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
+| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
+| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
+| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
+| Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`
`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` |
+# Index
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
-| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
-| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
-| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
+| Index Apply Elapsed Time | `histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. | `prometheus` | `s` | `{{type}}-p95` |
+| Index Create Elapsed Time | `histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. | `prometheus` | `s` | `{{type}}-{{stage}}-p95` |
+| Index Create Rows and Bytes | `sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))`
`sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))` | `timeseries` | Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. | `prometheus` | `rowsps` | `{{type}}-rows` |
+| Index Memory Usage | `greptime_index_apply_memory_usage`
`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` |
+| Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` |
+| Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` |
+| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
# Metasrv
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
+| Inactive and Lease-expired Regions | `sum(greptime_meta_inactive_regions)`
`sum(greptime_lease_expired_region{instance=~"$datanode"})` | `timeseries` | Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. | `prometheus` | `short` | `inactive-regions` |
+| Heartbeat Health | `sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))`
`sum(greptime_meta_heartbeat_connection_num)`
`sum(rate(greptime_frontend_heartbeat_send_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_frontend_heartbeat_recv_count{instance=~"$frontend"}[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_send_count{instance=~"$datanode"}[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_recv_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. | `prometheus` | `short` | `meta-recv-rate` |
| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` |
-| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
+| Region migration error | `rate(greptime_meta_region_migration_error[$__rate_interval])` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` |
| Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` |
-| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
+| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))`
`sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
| Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) (
rate(greptime_meta_handler_execute_bucket[$__rate_interval])
-))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` |
-| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` |
-| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` |
-| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
-| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
-| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
-| Reconciliation stats | `greptime_meta_reconciliation_stats` | `timeseries` | Reconciliation stats | `prometheus` | `s` | `{{pod}}-{{table_type}}-{{type}}` |
-| Reconciliation steps | `histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
-# Flownode
-| Title | Query | Type | Description | Datasource | Unit | Legend Format |
-| --- | --- | --- | --- | --- | --- | --- |
-| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
-| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
-| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
-| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
-| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
-# Trigger
-| Title | Query | Type | Description | Datasource | Unit | Legend Format |
-| --- | --- | --- | --- | --- | --- | --- |
-| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
-| Trigger Eval Elapsed | `histogram_quantile(0.99,
- rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
-)`
`histogram_quantile(0.75,
- rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
-)` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
-| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
-| Send Alert Elapsed | `histogram_quantile(0.99,
- rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
-)`
`histogram_quantile(0.75,
- rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
-)` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99` |
-| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
-| Save Alert Elapsed | `histogram_quantile(0.99,
- rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
-)`
`histogram_quantile(0.75,
- rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
-)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` |
-| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
+))`
`sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` |
+| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` |
+| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Rate of heartbeats received by metasrv from datanodes and frontends. | `prometheus` | `s` | `{{pod}}` |
+| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))`
`sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average latency of metasrv key-value store operations by op and target. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
+| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Rate of metasrv key-value store operations by op. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
+| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))` | `timeseries` | p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
+| Reconciliation stats | `rate(greptime_meta_reconciliation_stats[$__rate_interval])` | `timeseries` | Reconciliation stats | `prometheus` | `ops` | `{{pod}}-{{table_type}}-{{type}}` |
+| Reconciliation steps | `histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
# Hotspot
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
@@ -240,3 +250,45 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
| Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
| Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
+# Object Store
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Read P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read\|Reader::read"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Write P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
+| List P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))` | `timeseries` | List P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
+| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Other Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
+# WAL
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
+| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
+| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
+| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
+| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
+| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
+| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
+# Flownode
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
+| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
+| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
+| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
+| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
+# Trigger
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
+| Trigger Eval Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `p99` |
+| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
+| Send Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `p99` |
+| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
+| Save Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `p99` |
+| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
diff --git a/grafana/dashboards/metrics/cluster/dashboard.yaml b/grafana/dashboards/metrics/cluster/dashboard.yaml
index ca8f88267f..127a8d606b 100644
--- a/grafana/dashboards/metrics/cluster/dashboard.yaml
+++ b/grafana/dashboards/metrics/cluster/dashboard.yaml
@@ -29,24 +29,36 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: __auto
- - title: Total Storage Size
+ - title: Total Query Rate
type: stat
- description: Total number of data file size.
- unit: decbytes
+ description: Total query API call rate across MySQL, PostgreSQL, and PromQL frontends.
+ unit: reqps
queries:
- - expr: select SUM(disk_size) from information_schema.region_statistics;
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
datasource:
- type: mysql
- uid: ${information_schema}
- - title: Total Rows
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: queries
+ - title: User-facing Error Rate
type: stat
- description: Total number of data rows in the cluster. Calculated by sum of rows from each region.
- unit: sishort
+ description: Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures.
+ unit: eps
queries:
- - expr: select SUM(region_rows) from information_schema.region_statistics;
+ - expr: sum(rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))
datasource:
- type: mysql
- uid: ${information_schema}
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: errors
+ - title: Recent Restarts
+ type: stat
+ description: Process restarts over the selected time range across GreptimeDB roles.
+ unit: short
+ queries:
+ - expr: sum(changes(process_start_time_seconds[$__range]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: restarts
- title: Deployment
type: stat
description: The deployment topology of GreptimeDB.
@@ -87,6 +99,24 @@ groups:
datasource:
type: mysql
uid: ${information_schema}
+ - title: Total Storage Size
+ type: stat
+ description: Total number of data file size.
+ unit: decbytes
+ queries:
+ - expr: select SUM(disk_size) from information_schema.region_statistics;
+ datasource:
+ type: mysql
+ uid: ${information_schema}
+ - title: Total Rows
+ type: stat
+ description: Total number of data rows in the cluster. Calculated by sum of rows from each region.
+ unit: sishort
+ queries:
+ - expr: select SUM(region_rows) from information_schema.region_statistics;
+ datasource:
+ type: mysql
+ uid: ${information_schema}
- title: Data Size
type: stat
description: The data size of wal/index/manifest in the GreptimeDB.
@@ -104,6 +134,56 @@ groups:
datasource:
type: mysql
uid: ${information_schema}
+ - title: Total Ingestion Rate Trend
+ type: timeseries
+ description: Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row.
+ unit: rowsps
+ queries:
+ - expr: sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: ingestion
+ - title: Total Query Rate Trend
+ type: timeseries
+ description: Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row.
+ unit: reqps
+ queries:
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: queries
+ - title: HTTP Request P99 and Avg
+ type: timeseries
+ description: Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-p99
+ - expr: sum(rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-avg
+ - title: gRPC Request P99 and Avg
+ type: timeseries
+ description: Tail and average latency for gRPC requests served by frontends.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-p99
+ - expr: sum(rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-avg
- title: Ingestion
panels:
- title: Total Ingestion Rate
@@ -123,71 +203,478 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: ingestion
- - title: Ingestion Rate by Type
+ - title: Ingestion Rate by Protocol
type: timeseries
- description: |
- Total ingestion rate.
-
- Here we listed 3 primary protocols:
-
- - Prometheus remote write
- - Greptime's gRPC API (when using our ingest SDK)
- - Log ingestion http API
+ description: Rows, samples, or documents ingested by primary observability and table-ingestion protocols.
unit: rowsps
queries:
- - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))
+ - expr: sum(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: http-logs
- - expr: sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))
+ legendFormat: table-operator
+ - expr: sum(rate(greptime_servers_prometheus_remote_write_samples{instance=~"$frontend"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: prometheus-remote-write
- - title: Queries
- panels:
- - title: Total Query Rate
+ - expr: sum(rate(greptime_servers_http_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-logs
+ - expr: sum(rate(greptime_servers_loki_logs_ingestion_counter{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: loki-logs
+ - expr: sum(rate(greptime_servers_elasticsearch_logs_docs_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: elasticsearch-docs
+ - expr: sum(rate(greptime_frontend_otlp_metrics_rows{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-metrics
+ - expr: sum(rate(greptime_frontend_otlp_logs_rows{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-logs
+ - expr: sum(rate(greptime_frontend_otlp_traces_rows{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-traces
+ - title: Ingestion Latency by Protocol
type: timeseries
- description: |-
- Total rate of query API calls by protocol. This metric is collected from frontends.
-
- Here we listed 3 main protocols:
- - MySQL
- - Postgres
- - Prometheus API
-
- Note that there are some other minor query APIs like /sql are not included
- unit: reqps
+ description: p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints.
+ unit: s
queries:
- - expr: sum (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: mysql
- - expr: sum (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ legendFormat: prometheus-write
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: pg
- - expr: sum (rate(greptime_servers_http_promql_elapsed_counte{instance=~"$frontend"}[$__rate_interval]))
+ legendFormat: http-logs
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: promql
- - title: Resources
- panels:
- - title: Datanode Memory per Instance
+ legendFormat: loki-logs
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: elasticsearch
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-metrics
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-logs
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-traces
+ - expr: sum(rate(greptime_servers_http_prometheus_write_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: prom-write-avg
+ - expr: sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-logs-avg
+ - expr: sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: loki-logs-avg
+ - expr: sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: elasticsearch-avg
+ - expr: sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-metrics-avg
+ - expr: sum(rate(greptime_servers_http_otlp_logs_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-logs-avg
+ - expr: sum(rate(greptime_servers_http_otlp_traces_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-traces-avg
+ - title: Bulk Insert Message Rows and Size
type: timeseries
- description: Current memory usage by instance
+ description: Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency.
+ unit: rowsps
+ queries:
+ - expr: sum(rate(greptime_table_operator_bulk_insert_message_rows_sum{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: rows
+ - expr: sum(rate(greptime_table_operator_bulk_insert_message_size_sum{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: bytes
+ - title: Prom Store Flush Pipeline
+ type: timeseries
+ description: Remote-write pending-row flush operations, flushed rows, and p99 flush latency.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_prom_store_flush_total{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-ops
+ - expr: sum(rate(greptime_prom_store_flush_rows_sum{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-rows
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-p99
+ - title: OTLP Trace Failures
+ type: timeseries
+ description: OTLP trace ingestion failures reported by frontends.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_frontend_otlp_traces_failure_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trace-failures
+ - title: Health
+ panels:
+ - title: Protocol Error Rates
+ type: timeseries
+ description: User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related.
+ unit: eps
+ queries:
+ - expr: sum by (protocol) (rate(greptime_servers_error{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: server-{{protocol}}
+ - expr: sum by (code) (rate(greptime_servers_auth_failure_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: auth-{{code}}
+ - expr: sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics",code!~"2.."}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-{{method}}-{{path}}-{{code}}
+ - expr: sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend",code!~"0|OK"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-{{path}}-{{code}}
+ - title: Frontend and Query Rejections
+ type: timeseries
+ description: Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution.
+ unit: rps
+ queries:
+ - expr: sum(rate(greptime_servers_request_memory_rejected_total{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: request-memory
+ - expr: sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: query-memory
+ - title: Datanode Write Failures
+ type: timeseries
+ description: Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing.
+ unit: eps
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: region-request-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: failed-insert-[{{instance}}]-[{{pod}}]
+ - title: Buffered Ingestion Loss
+ type: timeseries
+ description: Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_pending_rows_flush_failures{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-failures
+ - expr: sum(rate(greptime_pending_rows_flush_dropped_rows{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: dropped-rows
+ - title: Mito Backpressure and Failures
+ type: timeseries
+ description: Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: write-reject
+ - expr: sum(rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: write-stall
+ - expr: sum(rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-failure
+ - expr: sum(rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-failure
+ - title: Scan and Compaction Memory Rejects
+ type: timeseries
+ description: Datanode scan and compaction memory rejection/exhaustion counters.
+ unit: rps
+ queries:
+ - expr: sum(rate(greptime_mito_scan_requests_rejected_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: scan-rejected
+ - expr: sum(rate(greptime_mito_scan_memory_exhausted_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: scan-exhausted
+ - expr: sum(rate(greptime_mito_compaction_memory_rejected_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-rejected
+ - title: OpenDAL Errors
+ type: timeseries
+ description: Object-store errors by scheme, operation, and error, excluding NotFound noise.
+ unit: eps
+ queries:
+ - expr: sum by (scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode",error!="NotFound"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{scheme}}-{{operation}}-{{error}}'
+ - title: Metasrv Failures
+ type: timeseries
+ description: Region migration and reconciliation failures in metasrv.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: migration-fail
+ - expr: sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: reconciliation-error
+ - title: Flow and Trigger Failures
+ type: timeseries
+ description: Derived-data and alerting pipeline failures.
+ unit: eps
+ queries:
+ - expr: sum by (code) (rate(greptime_flow_errors[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flow-{{code}}
+ - expr: sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trigger-evaluate
+ - expr: sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trigger-send-alert
+ - expr: sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trigger-save-alert
+ - title: Mito GC Failures
+ type: timeseries
+ description: Mito garbage-collection errors and skipped/orphaned files on datanodes.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: gc-errors
+ - expr: sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: orphaned-index-files
+ - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: skipped-unparsable
+ - title: Capacity
+ panels:
+ - title: Runtime Threads
+ type: timeseries
+ description: Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_runtime_threads_alive)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: alive-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_runtime_threads_idle)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: idle-[{{instance}}]-[{{pod}}]
+ - title: Request Memory Utilization
+ type: timeseries
+ description: Frontend request memory usage divided by configured request memory limit.
+ unit: percentunit
+ queries:
+ - expr: sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes{instance=~"$frontend"}) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes{instance=~"$frontend"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Query Memory Usage
+ type: timeseries
+ description: Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation.
unit: bytes
queries:
- - expr: sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)
+ - expr: sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{ pod }}]'
- - expr: max(greptime_memory_limit_in_bytes{instance=~"$datanode"})
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Scan and Compaction Memory
+ type: timeseries
+ description: Datanode scan memory usage and compaction memory utilization.
+ unit: bytes
+ queries:
+ - expr: sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: scan-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-used-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-limit-[{{instance}}]-[{{pod}}]
+ - title: Write Buffer and Active Stalling
+ type: timeseries
+ description: Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure.
+ unit: bytes
+ queries:
+ - expr: sum by (instance, pod) (greptime_mito_write_buffer_bytes{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: buffer-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: stalling-[{{instance}}]-[{{pod}}]
+ - title: Prom Store Backlog
+ type: timeseries
+ description: Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_prom_store_pending_rows{instance=~"$frontend"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: rows-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_prom_store_pending_batches{instance=~"$frontend"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: batches-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_prom_store_pending_workers{instance=~"$frontend"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: workers-[{{instance}}]-[{{pod}}]
+ - title: Inflight Flush and Compaction
+ type: timeseries
+ description: Current in-flight flush and compaction tasks on datanodes.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_mito_inflight_flush_count{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_inflight_compaction_count{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-[{{instance}}]-[{{pod}}]
+ - title: Resources
+ panels:
+ - title: Frontend CPU Usage per Instance
+ type: timeseries
+ description: Current cpu usage by instance
+ unit: none
+ queries:
+ - expr: sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu'
+ - expr: max(greptime_cpu_limit_in_millicores{instance=~"$frontend"})
datasource:
type: prometheus
uid: ${metrics}
@@ -207,6 +694,21 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: limit
+ - title: Metasrv CPU Usage per Instance
+ type: timeseries
+ description: Current cpu usage by instance
+ unit: none
+ queries:
+ - expr: sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{ pod }}]'
+ - expr: max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: limit
- title: Frontend Memory per Instance
type: timeseries
description: Current memory usage by instance
@@ -222,17 +724,17 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: limit
- - title: Frontend CPU Usage per Instance
+ - title: Datanode Memory per Instance
type: timeseries
- description: Current cpu usage by instance
- unit: none
+ description: Current memory usage by instance
+ unit: bytes
queries:
- - expr: sum(rate(process_cpu_seconds_total{instance=~"$frontend"}[$__rate_interval]) * 1000) by (instance, pod)
+ - expr: sum(process_resident_memory_bytes{instance=~"$datanode"}) by (instance, pod)
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu'
- - expr: max(greptime_cpu_limit_in_millicores{instance=~"$frontend"})
+ legendFormat: '[{{instance}}]-[{{ pod }}]'
+ - expr: max(greptime_memory_limit_in_bytes{instance=~"$datanode"})
datasource:
type: prometheus
uid: ${metrics}
@@ -252,17 +754,17 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: limit
- - title: Metasrv CPU Usage per Instance
+ - title: Flownode CPU Usage per Instance
type: timeseries
description: Current cpu usage by instance
unit: none
queries:
- - expr: sum(rate(process_cpu_seconds_total{instance=~"$metasrv"}[$__rate_interval]) * 1000) by (instance, pod)
+ - expr: sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_cpu_limit_in_millicores{instance=~"$metasrv"})
+ - expr: max(greptime_cpu_limit_in_millicores{instance=~"$flownode"})
datasource:
type: prometheus
uid: ${metrics}
@@ -282,21 +784,168 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: limit
- - title: Flownode CPU Usage per Instance
+ - title: Queries
+ panels:
+ - title: Query Rate by Protocol
type: timeseries
- description: Current cpu usage by instance
- unit: none
+ description: Query API call rates by protocol, collected from frontends.
+ unit: reqps
queries:
- - expr: sum(rate(process_cpu_seconds_total{instance=~"$flownode"}[$__rate_interval]) * 1000) by (instance, pod)
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_cpu_limit_in_millicores{instance=~"$flownode"})
+ legendFormat: mysql
+ - expr: sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: limit
+ legendFormat: postgres
+ - expr: sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql
+ - expr: sum(rate(greptime_servers_http_sql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-sql
+ - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-query
+ - title: Query Latency by Protocol
+ type: timeseries
+ description: p95, p99, and average query latency by main frontend protocol.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-p95
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-p99
+ - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-p95
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-p99
+ - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql-p95
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql-p99
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-avg
+ - expr: sum(rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-avg
+ - expr: sum(rate(greptime_servers_http_promql_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql-avg
+ - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-query-avg
+ - title: Query Stage Latency
+ type: timeseries
+ description: p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p95-{{stage}}
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99-{{stage}}
+ - title: Merge Scan Fan-out and Errors
+ type: timeseries
+ description: Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_query_merge_scan_regions)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: regions-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: errors-[{{instance}}]-[{{pod}}]
+ - title: Pushdown Fallback Errors
+ type: timeseries
+ description: Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: pushdown-fallback-errors
+ - title: PromQL Series Count
+ type: timeseries
+ description: Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_promql_series_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Connections and Prepared Statements
+ type: timeseries
+ description: MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_servers_mysql_connection_count{instance=~"$frontend"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-connections-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_servers_postgres_connection_count{instance=~"$frontend"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-connections-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-prepared-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-prepared-[{{instance}}]-[{{pod}}]
- title: Frontend Requests
panels:
- title: HTTP QPS per Instance
@@ -309,9 +958,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]'
- - title: HTTP P99 per Instance
+ - title: HTTP P99 and Avg per Instance
type: timeseries
- description: HTTP P99 per Instance.
+ description: HTTP P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])))
@@ -319,6 +968,11 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99'
+ - expr: sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{instance=~"$frontend",path!~"/health|/metrics"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg'
- title: gRPC QPS per Instance
type: timeseries
description: gRPC QPS per Instance.
@@ -329,9 +983,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]'
- - title: gRPC P99 per Instance
+ - title: gRPC P99 and Avg per Instance
type: timeseries
- description: gRPC P99 per Instance.
+ description: gRPC P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
@@ -339,6 +993,11 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99'
+ - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg'
- title: MySQL QPS per Instance
type: timeseries
description: MySQL QPS per Instance.
@@ -349,9 +1008,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: MySQL P99 per Instance
+ - title: MySQL P99 and Avg per Instance
type: timeseries
- description: MySQL P99 per Instance.
+ description: MySQL P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
@@ -359,6 +1018,11 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]-p99'
+ - expr: sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
- title: PostgreSQL QPS per Instance
type: timeseries
description: PostgreSQL QPS per Instance.
@@ -369,9 +1033,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: PostgreSQL P99 per Instance
+ - title: PostgreSQL P99 and Avg per Instance
type: timeseries
- description: PostgreSQL P99 per Instance.
+ description: PostgreSQL P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{instance=~"$frontend"}[$__rate_interval])))
@@ -379,18 +1043,13 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - title: Frontend to Datanode
- panels:
- - title: Ingest Rows per Instance
- type: timeseries
- description: Ingestion rate by row as in each frontend
- unit: rowsps
- queries:
- - expr: sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{instance=~"$frontend"}[$__rate_interval]))
+ - expr: sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count{instance=~"$frontend"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
+ - title: Frontend to Datanode
+ panels:
- title: Region Call QPS per Instance
type: timeseries
description: Region Call QPS per Instance.
@@ -401,9 +1060,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]'
- - title: Region Call P99 per Instance
+ - title: Region Call P99 and Avg per Instance
type: timeseries
- description: Region Call P99 per Instance.
+ description: Region Call P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{instance=~"$frontend"}[$__rate_interval])))
@@ -411,7 +1070,12 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]'
- - title: 'Frontend Handle Bulk Insert Elapsed Time '
+ - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum{instance=~"$frontend"}[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg'
+ - title: Frontend Handle Bulk Insert Elapsed Time
type: timeseries
description: Per-stage time for frontend to handle bulk insert requests
unit: s
@@ -426,7 +1090,104 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95'
- - title: Mito Engine
+ - title: Datanode
+ panels:
+ - title: Region Request Failures and Failed Inserts
+ type: timeseries
+ description: Datanode region request failures and failed inserts by instance.
+ unit: eps
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: request-fail-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: failed-insert-[{{instance}}]-[{{pod}}]
+ - title: Write Rejects and Stalls
+ type: timeseries
+ description: Mito write rejects, write stall events, and active write stalling by datanode.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_mito_write_reject_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: reject-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_mito_write_stall_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: stall-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_write_stalling_count{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: active-stalling-[{{instance}}]-[{{pod}}]
+ - title: Flush and Compaction Failures
+ type: timeseries
+ description: Mito flush and compaction failure rates by datanode.
+ unit: eps
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_mito_flush_failure_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_mito_compaction_failure_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-[{{instance}}]-[{{pod}}]
+ - title: Mito GC Health
+ type: timeseries
+ description: Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_mito_gc_runs_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: runs
+ - expr: sum(rate(greptime_mito_gc_errors_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: errors
+ - expr: sum(rate(greptime_mito_gc_files_deleted_total{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: files-deleted
+ - expr: sum(rate(greptime_mito_gc_orphaned_index_files{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: orphaned-index-files
+ - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: skipped-unparsable
+ - title: Mito GC Duration
+ type: timeseries
+ description: P99 and average Mito garbage-collection duration by stage.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket{instance=~"$datanode"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}-p99'
+ - expr: sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}-avg'
+ - title: Storage
panels:
- title: Request OPS per Instance
type: timeseries
@@ -438,9 +1199,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
- - title: Request P99 per Instance
+ - title: Request P99 and Avg per Instance
type: timeseries
- description: Request P99 per Instance.
+ description: Request P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
@@ -448,6 +1209,31 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
+ - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg'
+ - title: Request Wait P99 and Avg per Worker
+ type: timeseries
+ description: Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p95'
+ - expr: histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket{instance=~"$datanode"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p99'
+ - expr: sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-avg'
- title: Write Buffer per Instance
type: timeseries
description: Write Buffer per Instance.
@@ -468,25 +1254,6 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Flush OPS per Instance
- type: timeseries
- description: Flush QPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]'
- - title: Write Stall per Instance
- type: timeseries
- description: Write Stall per Instance.
- queries:
- - expr: sum by(instance, pod) (greptime_mito_write_stall_total{instance=~"$datanode"})
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- title: Read Stage OPS per Instance
type: timeseries
description: Read Stage OPS per Instance.
@@ -497,9 +1264,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Read Stage P99 per Instance
+ - title: Read Stage P99 and Avg per Instance
type: timeseries
- description: Read Stage P99 per Instance.
+ description: Read Stage P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
@@ -507,9 +1274,14 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]'
- - title: Write Stage P99 per Instance
+ - expr: sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
+ - title: Write Stage P99 and Avg per Instance
type: timeseries
- description: Write Stage P99 per Instance.
+ description: Write Stage P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
@@ -517,61 +1289,11 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]'
- - title: Compaction OPS per Instance
- type: timeseries
- description: Compaction OPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{pod}}]'
- - title: Compaction Elapsed Time per Instance by Stage
- type: timeseries
- description: Compaction latency by stage
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99'
- - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ - expr: sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
- - title: Compaction P99 per Instance
- type: timeseries
- description: Compaction P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction'
- - title: WAL write size
- type: timeseries
- description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.
- unit: bytes
- queries:
- - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95'
- - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99'
- - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-throughput'
- title: Cached Bytes per Instance
type: timeseries
description: Cached Bytes per Instance.
@@ -582,61 +1304,6 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
- - title: Inflight Compaction
- type: timeseries
- description: Ongoing compaction task count
- unit: none
- queries:
- - expr: greptime_mito_inflight_compaction_count
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: WAL sync duration seconds
- type: timeseries
- description: Raft engine (local disk) log store sync latency, p99
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - title: Log Store op duration seconds
- type: timeseries
- description: Write-ahead log operations latency at p99
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99'
- - title: Inflight Flush
- type: timeseries
- description: Ongoing flush task count
- unit: none
- queries:
- - expr: greptime_mito_inflight_flush_count
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Compaction Input/Output Bytes
- type: timeseries
- description: Compaction oinput output bytes
- unit: bytes
- queries:
- - expr: sum by(instance, pod) (greptime_mito_compaction_input_bytes)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-input'
- - expr: sum by(instance, pod) (greptime_mito_compaction_output_bytes)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-output'
- title: Region Worker Handle Bulk Insert Requests
type: timeseries
description: Per-stage elapsed time for region worker to handle bulk insert region requests.
@@ -654,7 +1321,7 @@ groups:
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG'
- title: Active Series and Field Builders Count
type: timeseries
- description: Compaction oinput output bytes
+ description: Active series and field-builder counts per memtable by instance.
unit: none
queries:
- expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count)
@@ -691,161 +1358,282 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
- - title: OpenDAL
+ - title: Flush and Compaction
panels:
- - title: QPS per Instance
+ - title: Flush OPS per Instance
type: timeseries
- description: QPS per Instance.
+ description: Flush QPS per Instance.
unit: ops
queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))
+ - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{instance=~"$datanode"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Read QPS per Instance
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]'
+ - title: Flush Elapsed Time
type: timeseries
- description: Read QPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Read P99 per Instance
- type: timeseries
- description: Read P99 per Instance.
+ description: Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read|Reader::read"}[$__rate_interval])))
+ - expr: histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Write QPS per Instance
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95'
+ - expr: histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
+ - expr: sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg'
+ - title: Flush Throughput
type: timeseries
- description: Write QPS per Instance.
- unit: ops
+ description: Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs.
+ unit: Bps
queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+ - expr: sum by (instance, pod) (rate(greptime_mito_flush_bytes_total{instance=~"$datanode"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Write P99 per Instance
- type: timeseries
- description: Write P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write|Writer::close|write"}[$__rate_interval])))
+ legendFormat: '[{{instance}}]-[{{pod}}]-bytes'
+ - expr: sum by (instance, pod) (rate(greptime_mito_flush_file_total{instance=~"$datanode"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: List QPS per Instance
+ legendFormat: '[{{instance}}]-[{{pod}}]-files'
+ - title: Inflight Flush
type: timeseries
- description: List QPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
- - title: List P99 per Instance
- type: timeseries
- description: List P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
- - title: Other Requests per Instance
- type: timeseries
- description: Other Requests per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode",operation!~"read|write|list|stat"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Other Request P99 per Instance
- type: timeseries
- description: Other Request P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|write|list|Writer::write|Writer::close|Reader::read"}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Opendal traffic
- type: timeseries
- description: Total traffic as in bytes by instance and operation
- unit: decbytes
- queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: OpenDAL errors per Instance
- type: timeseries
- description: OpenDAL error counts per Instance.
- queries:
- - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]'
- - title: Remote WAL
- panels:
- - title: Triggered region flush total
- type: timeseries
- description: Triggered region flush total
+ description: Ongoing flush task count
unit: none
queries:
- - expr: meta_triggered_region_flush_total
+ - expr: greptime_mito_inflight_flush_count
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{topic_name}}'
- - title: Triggered region checkpoint total
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Compaction OPS per Instance
type: timeseries
- description: Triggered region checkpoint total
+ description: Compaction OPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{pod}}]'
+ - title: Inflight Compaction
+ type: timeseries
+ description: Ongoing compaction task count
unit: none
queries:
- - expr: meta_triggered_region_checkpoint_total
+ - expr: greptime_mito_inflight_compaction_count
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{topic_name}}'
- - title: Topic estimated replay size
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Compaction P99 and Avg per Instance
type: timeseries
- description: Topic estimated max replay size
+ description: Compaction P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p99'
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum{instance=~"$datanode"}[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
+ - title: Compaction Elapsed Time per Instance by Stage
+ type: timeseries
+ description: Compaction latency by stage
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{instance=~"$datanode"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99'
+ - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{instance=~"$datanode"}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
+ - title: Compaction Input/Output Bytes
+ type: timeseries
+ description: Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume.
+ unit: Bps
+ queries:
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-input'
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-output'
+ - title: Index
+ panels:
+ - title: Index Apply Elapsed Time
+ type: timeseries
+ description: Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-p95'
+ - expr: histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-p99'
+ - title: Index Create Elapsed Time
+ type: timeseries
+ description: Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-{{stage}}-p95'
+ - expr: histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-{{stage}}-p99'
+ - title: Index Create Rows and Bytes
+ type: timeseries
+ description: Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure.
+ unit: rowsps
+ queries:
+ - expr: sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-rows'
+ - expr: sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-bytes'
+ - title: Index Memory Usage
+ type: timeseries
+ description: Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work.
unit: bytes
queries:
- - expr: meta_topic_estimated_replay_size
+ - expr: greptime_index_apply_memory_usage
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{topic_name}}'
- - title: Kafka logstore's bytes traffic
+ legendFormat: apply
+ - expr: sum by (type) (greptime_index_create_memory_usage)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: create-{{type}}
+ - title: Index IO Bytes
type: timeseries
- description: Kafka logstore's bytes traffic
- unit: bytes
+ description: Index read and write byte rates by operation and file type for puffin and intermediate files.
+ unit: Bps
queries:
- - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])
+ - expr: sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{logstore}}'
+ legendFormat: '{{type}}-{{file_type}}'
+ - title: Index IO Operations
+ type: timeseries
+ description: Index IO operation rates by operation and file type, including read, write, seek, and flush operations.
+ unit: ops
+ queries:
+ - expr: sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-{{file_type}}'
+ - title: Index Cache
+ type: timeseries
+ description: Index-related cache hits, misses, and evictions from Mito caches.
+ unit: ops
+ queries:
+ - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: hit-{{type}}
+ - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: miss-{{type}}
+ - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: evict-{{type}}-{{cause}}
- title: Metasrv
panels:
+ - title: Inactive and Lease-expired Regions
+ type: timeseries
+ description: Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues.
+ unit: short
+ queries:
+ - expr: sum(greptime_meta_inactive_regions)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: inactive-regions
+ - expr: sum(greptime_lease_expired_region{instance=~"$datanode"})
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: lease-expired
+ - title: Heartbeat Health
+ type: timeseries
+ description: Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: meta-recv-rate
+ - expr: sum(greptime_meta_heartbeat_connection_num)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: meta-connections
+ - expr: sum(rate(greptime_frontend_heartbeat_send_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: frontend-send
+ - expr: sum(rate(greptime_frontend_heartbeat_recv_count{instance=~"$frontend"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: frontend-recv
+ - expr: sum(rate(greptime_datanode_heartbeat_send_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-send
+ - expr: sum(rate(greptime_datanode_heartbeat_recv_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-recv
- title: Region migration datanode
type: status-history
description: Counter of region migration by source and destination
@@ -865,7 +1653,7 @@ groups:
description: Counter of region migration error
unit: none
queries:
- - expr: greptime_meta_region_migration_error
+ - expr: rate(greptime_meta_region_migration_error[$__rate_interval])
datasource:
type: prometheus
uid: ${metrics}
@@ -900,6 +1688,11 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}} {{op}} {{type}} {{result}} p90'
+ - expr: sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg'
- title: Handler Execution Latency
type: timeseries
description: |
@@ -914,20 +1707,25 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}} {{name}} p90'
+ - expr: sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{name}}]-avg'
- title: Heartbeat Packet Size
type: timeseries
description: |
Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
unit: bytes
queries:
- - expr: histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))
+ - expr: histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}}'
- title: Meta Heartbeat Receive Rate
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: Rate of heartbeats received by metasrv from datanodes and frontends.
unit: s
queries:
- expr: rate(greptime_meta_heartbeat_rate[$__rate_interval])
@@ -937,17 +1735,22 @@ groups:
legendFormat: '{{pod}}'
- title: Meta KV Ops Latency
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: p99 and average latency of metasrv key-value store operations by op and target.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))
+ - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}}-{{op}} p99'
+ - expr: sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{op}}]-[{{target}}]-avg'
- title: Rate of meta KV Ops
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: Rate of metasrv key-value store operations by op.
unit: none
queries:
- expr: rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])
@@ -957,35 +1760,35 @@ groups:
legendFormat: '{{pod}}-{{op}} p99'
- title: DDL Latency
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step.
unit: s
queries:
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateLogicalTables-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateTable-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateView-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateFlow-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: DropTable-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
@@ -993,9 +1796,9 @@ groups:
- title: Reconciliation stats
type: timeseries
description: Reconciliation stats
- unit: s
+ unit: ops
queries:
- - expr: greptime_meta_reconciliation_stats
+ - expr: rate(greptime_meta_reconciliation_stats[$__rate_interval])
datasource:
type: prometheus
uid: ${metrics}
@@ -1005,154 +1808,11 @@ groups:
description: Elapsed of Reconciliation steps
unit: s
queries:
- - expr: histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)
+ - expr: histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '{{procedure_name}}-{{step}}-P90'
- - title: Flownode
- panels:
- - title: Flow Ingest / Output Rate
- type: timeseries
- description: Flow Ingest / Output Rate.
- queries:
- - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]'
- - title: Flow Ingest Latency
- type: timeseries
- description: Flow Ingest Latency.
- queries:
- - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p95'
- - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - title: Flow Operation Latency
- type: timeseries
- description: Flow Operation Latency.
- queries:
- - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95'
- - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
- - title: Flow Buffer Size per Instance
- type: timeseries
- description: Flow Buffer Size per Instance.
- queries:
- - expr: greptime_flow_input_buf_size
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Flow Processing Error per Instance
- type: timeseries
- description: Flow Processing Error per Instance.
- queries:
- - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]'
- - title: Trigger
- panels:
- - title: Trigger Count
- type: timeseries
- description: Total number of triggers currently defined.
- queries:
- - expr: greptime_trigger_count{}
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- - title: Trigger Eval Elapsed
- type: timeseries
- description: Elapsed time for trigger evaluation, including query execution and condition evaluation.
- unit: s
- queries:
- - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p75'
- - title: Trigger Eval Failure Rate
- type: timeseries
- description: Rate of failed trigger evaluations.
- unit: none
- queries:
- - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval])
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- - title: Send Alert Elapsed
- type: timeseries
- description: Elapsed time to send trigger alerts to notification channels.
- unit: s
- queries:
- - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99'
- - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75'
- - title: Send Alert Failure Rate
- type: timeseries
- description: Rate of failures when sending trigger alerts.
- unit: none
- queries:
- - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval])
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- - title: Save Alert Elapsed
- type: timeseries
- description: Elapsed time to persist trigger alert records.
- unit: s
- queries:
- - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99'
- - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75'
- - title: Save Alert Failure Rate
- type: timeseries
- description: Rate of failures when persisting trigger alert records.
- unit: none
- queries:
- - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- title: Hotspot
panels:
- title: Hotspot Regions
@@ -1310,3 +1970,379 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '{{stage}}'
+ - title: Object Store
+ panels:
+ - title: QPS per Instance
+ type: timeseries
+ description: QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Read QPS per Instance
+ type: timeseries
+ description: Read QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Read P99 and Avg per Instance
+ type: timeseries
+ description: Read P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode",operation=~"read|Reader::read"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"read|Reader::read"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: Write QPS per Instance
+ type: timeseries
+ description: Write QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Write P99 and Avg per Instance
+ type: timeseries
+ description: Write P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation =~ "Writer::write|Writer::close|write"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: List QPS per Instance
+ type: timeseries
+ description: List QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
+ - title: List P99 and Avg per Instance
+ type: timeseries
+ description: List P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation="list"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation="list"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: Other Requests per Instance
+ type: timeseries
+ description: Other Requests per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Other Request P99 and Avg per Instance
+ type: timeseries
+ description: Other Request P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{instance=~"$datanode", operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: Opendal traffic
+ type: timeseries
+ description: Total traffic as in bytes by instance and operation
+ unit: decbytes
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{instance=~"$datanode"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: OpenDAL errors per Instance
+ type: timeseries
+ description: OpenDAL error counts per Instance.
+ queries:
+ - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{instance=~"$datanode", error!="NotFound"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]'
+ - title: WAL
+ panels:
+ - title: WAL write size
+ type: timeseries
+ description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.
+ unit: bytes
+ queries:
+ - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95'
+ - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99'
+ - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-throughput'
+ - title: WAL sync duration seconds
+ type: timeseries
+ description: Raft engine (local disk) log store sync latency, p99
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p99'
+ - title: Log Store op duration seconds
+ type: timeseries
+ description: Write-ahead log operations latency at p99
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99'
+ - title: Triggered region flush total
+ type: timeseries
+ description: Triggered region flush total
+ unit: none
+ queries:
+ - expr: meta_triggered_region_flush_total
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{topic_name}}'
+ - title: Triggered region checkpoint total
+ type: timeseries
+ description: Triggered region checkpoint total
+ unit: none
+ queries:
+ - expr: meta_triggered_region_checkpoint_total
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{topic_name}}'
+ - title: Topic estimated replay size
+ type: timeseries
+ description: Topic estimated max replay size
+ unit: bytes
+ queries:
+ - expr: meta_topic_estimated_replay_size
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{topic_name}}'
+ - title: Kafka logstore's bytes traffic
+ type: timeseries
+ description: Kafka logstore's bytes traffic
+ unit: bytes
+ queries:
+ - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{logstore}}'
+ - title: Flownode
+ panels:
+ - title: Flow Ingest / Output Rate
+ type: timeseries
+ description: Flow Ingest / Output Rate.
+ queries:
+ - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]'
+ - title: Flow Ingest Latency
+ type: timeseries
+ description: Flow Ingest Latency.
+ queries:
+ - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p95'
+ - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p99'
+ - expr: sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
+ - title: Flow Operation Latency
+ type: timeseries
+ description: Flow Operation Latency.
+ queries:
+ - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95'
+ - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
+ - title: Flow Buffer Size per Instance
+ type: timeseries
+ description: Flow Buffer Size per Instance.
+ queries:
+ - expr: greptime_flow_input_buf_size
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Flow Processing Error per Instance
+ type: timeseries
+ description: Flow Processing Error per Instance.
+ queries:
+ - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]'
+ - title: Trigger
+ panels:
+ - title: Trigger Count
+ type: timeseries
+ description: Total number of triggers currently defined.
+ queries:
+ - expr: greptime_trigger_count{}
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
+ - title: Trigger Eval Elapsed
+ type: timeseries
+ description: Elapsed time for trigger evaluation, including query execution and condition evaluation.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99
+ - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p75
+ - expr: sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: avg
+ - title: Trigger Eval Failure Rate
+ type: timeseries
+ description: Rate of failed trigger evaluations.
+ unit: none
+ queries:
+ - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
+ - title: Send Alert Elapsed
+ type: timeseries
+ description: Elapsed time to send trigger alerts to notification channels.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99
+ - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p75
+ - expr: sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: avg
+ - title: Send Alert Failure Rate
+ type: timeseries
+ description: Rate of failures when sending trigger alerts.
+ unit: none
+ queries:
+ - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
+ - title: Save Alert Elapsed
+ type: timeseries
+ description: Elapsed time to persist trigger alert records.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99
+ - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p75
+ - expr: sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: avg
+ - title: Save Alert Failure Rate
+ type: timeseries
+ description: Rate of failures when persisting trigger alert records.
+ unit: none
+ queries:
+ - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
diff --git a/grafana/dashboards/metrics/standalone/dashboard.json b/grafana/dashboards/metrics/standalone/dashboard.json
index c8313ecb7a..376e1665cd 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.json
+++ b/grafana/dashboards/metrics/standalone/dashboard.json
@@ -36,7 +36,7 @@
"x": 0,
"y": 0
},
- "id": 279,
+ "id": 393,
"panels": [],
"title": "Overview",
"type": "row"
@@ -68,12 +68,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 3,
"x": 0,
- "y": 1
+ "y": 1,
+ "w": 3,
+ "h": 4
},
- "id": 265,
+ "id": 380,
"options": {
"colorMode": "value",
"graphMode": "none",
@@ -142,12 +142,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 2,
"x": 3,
- "y": 1
+ "y": 1,
+ "w": 3,
+ "h": 4
},
- "id": 239,
+ "id": 381,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -227,12 +227,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
+ "x": 6,
+ "y": 1,
"w": 4,
- "x": 5,
- "y": 1
+ "h": 4
},
- "id": 249,
+ "id": 382,
"options": {
"colorMode": "value",
"graphMode": "none",
@@ -270,16 +270,18 @@
},
{
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
- "description": "Total number of data file size.",
+ "description": "Total query API call rate across MySQL, PostgreSQL, and PromQL frontends.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
+ "fieldMinMax": false,
"mappings": [],
+ "max": 2,
"thresholds": {
"mode": "absolute",
"steps": [
@@ -288,20 +290,20 @@
}
]
},
- "unit": "decbytes"
+ "unit": "reqps"
},
"overrides": []
},
"gridPos": {
- "h": 4,
+ "x": 10,
+ "y": 1,
"w": 4,
- "x": 9,
- "y": 1
+ "h": 4
},
- "id": 248,
+ "id": 383,
"options": {
"colorMode": "value",
- "graphMode": "area",
+ "graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
@@ -310,7 +312,8 @@
"lastNotNull"
],
"fields": "",
- "values": false
+ "limit": 1,
+ "values": true
},
"showPercentChange": false,
"textMode": "auto",
@@ -319,72 +322,62 @@
"pluginVersion": "11.6.0",
"targets": [
{
- "dataset": "information_schema",
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
"editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "select SUM(disk_size) from information_schema.region_statistics;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- }
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "queries",
+ "range": true,
+ "refId": "A"
}
],
- "title": "Total Storage Size",
+ "title": "Total Query Rate",
"type": "stat"
},
{
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
- "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.",
+ "description": "Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures.",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
+ "fieldMinMax": false,
"mappings": [],
+ "max": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
}
]
},
- "unit": "sishort"
+ "unit": "eps"
},
"overrides": []
},
"gridPos": {
- "h": 4,
+ "x": 14,
+ "y": 1,
"w": 4,
- "x": 13,
- "y": 1
+ "h": 4
},
- "id": 254,
+ "id": 384,
"options": {
"colorMode": "value",
- "graphMode": "area",
+ "graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
@@ -393,7 +386,8 @@
"lastNotNull"
],
"fields": "",
- "values": false
+ "limit": 1,
+ "values": true
},
"showPercentChange": false,
"textMode": "auto",
@@ -402,36 +396,94 @@
"pluginVersion": "11.6.0",
"targets": [
{
- "dataset": "information_schema",
"datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
+ "type": "prometheus",
+ "uid": "${metrics}"
},
"editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "select SUM(region_rows) from information_schema.region_statistics;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- }
+ "expr": "sum(rate(greptime_servers_error[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "errors",
+ "range": true,
+ "refId": "A"
}
],
- "title": "Total Rows",
+ "title": "User-facing Error Rate",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Process restarts over the selected time range across GreptimeDB roles.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "max": 2,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 18,
+ "y": 1,
+ "w": 6,
+ "h": 4
+ },
+ "id": 385,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "limit": 1,
+ "values": true
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(changes(process_start_time_seconds[$__range]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "restarts",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Recent Restarts",
"type": "stat"
},
{
@@ -462,12 +514,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 5,
"x": 0,
- "y": 5
+ "y": 5,
+ "w": 6,
+ "h": 4
},
- "id": 243,
+ "id": 386,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -634,12 +686,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 5,
- "x": 5,
- "y": 5
+ "x": 6,
+ "y": 5,
+ "w": 6,
+ "h": 4
},
- "id": 247,
+ "id": 387,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -782,6 +834,172 @@
"title": "Database Resources",
"type": "stat"
},
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Total number of data file size.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 5,
+ "w": 4,
+ "h": 4
+ },
+ "id": 388,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "select SUM(disk_size) from information_schema.region_statistics;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Total Storage Size",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Total number of data rows in the cluster. Calculated by sum of rows from each region.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "sishort"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 5,
+ "w": 4,
+ "h": 4
+ },
+ "id": 389,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "select SUM(region_rows) from information_schema.region_statistics;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Total Rows",
+ "type": "stat"
+ },
{
"datasource": {
"type": "mysql",
@@ -807,12 +1025,12 @@
"overrides": []
},
"gridPos": {
- "h": 4,
- "w": 5,
- "x": 10,
- "y": 5
+ "x": 20,
+ "y": 5,
+ "w": 4,
+ "h": 4
},
- "id": 278,
+ "id": 390,
"options": {
"colorMode": "value",
"graphMode": "area",
@@ -926,15 +1144,449 @@
"title": "Data Size",
"type": "stat"
},
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 9,
+ "w": 12,
+ "h": 6
+ },
+ "id": 538,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "ingestion",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Ingestion Rate Trend",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 9,
+ "w": 12,
+ "h": 6
+ },
+ "id": 539,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "queries",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Query Rate Trend",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 15,
+ "w": 12,
+ "h": 6
+ },
+ "id": 392,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~\"/health|/metrics\"}[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_requests_elapsed_sum{path!~\"/health|/metrics\"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "HTTP Request P99 and Avg",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Tail and average latency for gRPC requests served by frontends.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 15,
+ "w": 12,
+ "h": 6
+ },
+ "id": 557,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "gRPC Request P99 and Avg",
+ "type": "timeseries"
+ },
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 9
+ "y": 21
},
- "id": 275,
+ "id": 419,
"panels": [],
"title": "Ingestion",
"type": "row"
@@ -1001,12 +1653,12 @@
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 24,
"x": 0,
- "y": 10
+ "y": 22,
+ "w": 8,
+ "h": 6
},
- "id": 193,
+ "id": 413,
"options": {
"legend": {
"calcs": [
@@ -1030,7 +1682,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "ingestion",
@@ -1046,7 +1698,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Total ingestion rate.\n\nHere we listed 3 primary protocols:\n\n- Prometheus remote write\n- Greptime's gRPC API (when using our ingest SDK)\n- Log ingestion http API\n",
+ "description": "Rows, samples, or documents ingested by primary observability and table-ingestion protocols.",
"fieldConfig": {
"defaults": {
"color": {
@@ -1103,12 +1755,12 @@
"overrides": []
},
"gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 16
+ "x": 8,
+ "y": 22,
+ "w": 8,
+ "h": 6
},
- "id": 277,
+ "id": 414,
"options": {
"legend": {
"calcs": [
@@ -1132,12 +1784,12 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+ "expr": "sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))",
"hide": false,
"instant": false,
- "legendFormat": "http-logs",
+ "legendFormat": "table-operator",
"range": true,
- "refId": "http_logs"
+ "refId": "A"
},
{
"datasource": {
@@ -1150,10 +1802,4690 @@
"instant": false,
"legendFormat": "prometheus-remote-write",
"range": true,
- "refId": "prometheus-remote-write"
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "loki-logs",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "elasticsearch-docs",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-metrics",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-logs",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-traces",
+ "range": true,
+ "refId": "H"
}
],
- "title": "Ingestion Rate by Type",
+ "title": "Ingestion Rate by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 22,
+ "w": 8,
+ "h": 6
+ },
+ "id": 415,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "prometheus-write",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "loki-logs",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "elasticsearch",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-metrics",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-logs",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-traces",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "prom-write-avg",
+ "range": true,
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-logs-avg",
+ "range": true,
+ "refId": "I"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "loki-logs-avg",
+ "range": true,
+ "refId": "J"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "elasticsearch-avg",
+ "range": true,
+ "refId": "K"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-metrics-avg",
+ "range": true,
+ "refId": "L"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-logs-avg",
+ "range": true,
+ "refId": "M"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "otlp-traces-avg",
+ "range": true,
+ "refId": "N"
+ }
+ ],
+ "title": "Ingestion Latency by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "bytes"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "Bps"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 28,
+ "w": 8,
+ "h": 6
+ },
+ "id": 416,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "rows",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "bytes",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Bulk Insert Message Rows and Size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Remote-write pending-row flush operations, flushed rows, and p99 flush latency.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 28,
+ "w": 8,
+ "h": 6
+ },
+ "id": 417,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_prom_store_flush_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-ops",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-rows",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-p99",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Prom Store Flush Pipeline",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "OTLP trace ingestion failures reported by frontends.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 28,
+ "w": 8,
+ "h": 6
+ },
+ "id": 418,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trace-failures",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OTLP Trace Failures",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 34
+ },
+ "id": 403,
+ "panels": [],
+ "title": "Health",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 35,
+ "w": 8,
+ "h": 6
+ },
+ "id": 394,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (protocol) (rate(greptime_servers_error[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "server-{{protocol}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "auth-{{code}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\",code!~\"2..\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-{{method}}-{{path}}-{{code}}",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~\"0|OK\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-{{path}}-{{code}}",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Protocol Error Rates",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 35,
+ "w": 8,
+ "h": 6
+ },
+ "id": 395,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "request-memory",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "query-memory",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Frontend and Query Rejections",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 35,
+ "w": 8,
+ "h": 6
+ },
+ "id": 396,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "region-request-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Datanode Write Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 41,
+ "w": 8,
+ "h": 6
+ },
+ "id": 397,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_pending_rows_flush_failures[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-failures",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "dropped-rows",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Buffered Ingestion Loss",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 41,
+ "w": 8,
+ "h": 6
+ },
+ "id": 398,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_write_reject_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "write-reject",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_write_stall_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "write-stall",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_flush_failure_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-failure",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_compaction_failure_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-failure",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Mito Backpressure and Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Datanode scan and compaction memory rejection/exhaustion counters.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "rps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 41,
+ "w": 8,
+ "h": 6
+ },
+ "id": 399,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "scan-rejected",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "scan-exhausted",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-rejected",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Scan and Compaction Memory Rejects",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Object-store errors by scheme, operation, and error, excluding NotFound noise.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 47,
+ "w": 8,
+ "h": 6
+ },
+ "id": 400,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!=\"NotFound\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{scheme}}-{{operation}}-{{error}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OpenDAL Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region migration and reconciliation failures in metasrv.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 47,
+ "w": 8,
+ "h": 6
+ },
+ "id": 401,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "migration-fail",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "reconciliation-error",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Metasrv Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Derived-data and alerting pipeline failures.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 47,
+ "w": 8,
+ "h": 6
+ },
+ "id": 402,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (code) (rate(greptime_flow_errors[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flow-{{code}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trigger-evaluate",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trigger-send-alert",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "trigger-save-alert",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Flow and Trigger Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito garbage-collection errors and skipped/orphaned files on datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 53,
+ "w": 8,
+ "h": 6
+ },
+ "id": 550,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "gc-errors",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "orphaned-index-files",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "skipped-unparsable",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Mito GC Failures",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 59
+ },
+ "id": 411,
+ "panels": [],
+ "title": "Capacity",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 60,
+ "w": 8,
+ "h": 6
+ },
+ "id": 404,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_runtime_threads_alive)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "alive-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_runtime_threads_idle)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "idle-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Runtime Threads",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Frontend request memory usage divided by configured request memory limit.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 60,
+ "w": 8,
+ "h": 6
+ },
+ "id": 405,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Request Memory Utilization",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 60,
+ "w": 8,
+ "h": 6
+ },
+ "id": 406,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Query Memory Usage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Datanode scan memory usage and compaction memory utilization.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 66,
+ "w": 8,
+ "h": 6
+ },
+ "id": 407,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "scan-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-used-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-limit-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Scan and Compaction Memory",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 66,
+ "w": 8,
+ "h": 6
+ },
+ "id": 408,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_write_buffer_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "buffer-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "stalling-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Write Buffer and Active Stalling",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 66,
+ "w": 8,
+ "h": 6
+ },
+ "id": 409,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_prom_store_pending_rows)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "rows-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_prom_store_pending_batches)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "batches-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_prom_store_pending_workers)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "workers-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Prom Store Backlog",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current in-flight flush and compaction tasks on datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 72,
+ "w": 8,
+ "h": 6
+ },
+ "id": 410,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_inflight_flush_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_inflight_compaction_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Inflight Flush and Compaction",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 78
+ },
+ "id": 412,
+ "title": "Resources",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 79
+ },
+ "id": 465,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Last *",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Frontend CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 79
+ },
+ "id": 463,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Datanode CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 79
+ },
+ "id": 467,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Metasrv CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 87
+ },
+ "id": 464,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Frontend Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 87
+ },
+ "id": 462,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Datanode Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 87
+ },
+ "id": 466,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Metasrv Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current cpu usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 95
+ },
+ "id": 469,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "exemplar": false,
+ "expr": "max(greptime_cpu_limit_in_millicores)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flownode CPU Usage per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Current memory usage by instance",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 95
+ },
+ "id": 468,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true,
+ "sortBy": "Mean",
+ "sortDesc": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(process_resident_memory_bytes) by (instance, pod)",
+ "instant": false,
+ "legendFormat": "[{{ instance }}]-[{{ pod }}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "max(greptime_memory_limit_in_bytes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "limit",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flownode Memory per Instance",
+ "type": "timeseries"
+ },
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 103
+ },
+ "id": 427,
+ "panels": [],
+ "title": "Queries",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Query API call rates by protocol, collected from frontends.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 104,
+ "w": 8,
+ "h": 6
+ },
+ "id": 420,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "http-sql",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-query",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Query Rate by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "p95, p99, and average query latency by main frontend protocol.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 104,
+ "w": 8,
+ "h": 6
+ },
+ "id": 421,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-p95",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-p99",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql-p95",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql-p99",
+ "range": true,
+ "refId": "F"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-avg",
+ "range": true,
+ "refId": "G"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-avg",
+ "range": true,
+ "refId": "H"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "promql-avg",
+ "range": true,
+ "refId": "I"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "grpc-query-avg",
+ "range": true,
+ "refId": "J"
+ }
+ ],
+ "title": "Query Latency by Protocol",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 104,
+ "w": 8,
+ "h": 6
+ },
+ "id": 422,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p95-{{stage}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99-{{stage}}",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Query Stage Latency",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 110,
+ "w": 8,
+ "h": 6
+ },
+ "id": 423,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_query_merge_scan_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "regions-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "errors-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Merge Scan Fan-out and Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 110,
+ "w": 8,
+ "h": 6
+ },
+ "id": 424,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "pushdown-fallback-errors",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Pushdown Fallback Errors",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 110,
+ "w": 8,
+ "h": 6
+ },
+ "id": 425,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_promql_series_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "PromQL Series Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 116,
+ "w": 8,
+ "h": 6
+ },
+ "id": 426,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_servers_mysql_connection_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-connections-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_servers_postgres_connection_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-connections-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "mysql-prepared-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "postgres-prepared-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "D"
+ }
+ ],
+ "title": "Connections and Prepared Statements",
"type": "timeseries"
},
{
@@ -1162,1092 +6494,9 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 22
+ "y": 122
},
- "id": 276,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Total rate of query API calls by protocol. This metric is collected from frontends.\n\nHere we listed 3 main protocols:\n- MySQL\n- Postgres\n- Prometheus API\n\nNote that there are some other minor query APIs like /sql are not included",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "reqps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 246
- },
- "id": 255,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "mysql",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "pg",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "promql",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "Total Query Rate",
- "type": "timeseries"
- }
- ],
- "title": "Queries",
- "type": "row"
- },
- {
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 23
- },
- "id": 274,
- "panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 247
- },
- "id": 256,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Datanode Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 247
- },
- "id": 262,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Datanode CPU Usage per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 257
- },
- "id": 266,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Frontend Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 257
- },
- "id": 268,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Last *",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]-cpu",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Frontend CPU Usage per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 267
- },
- "id": 269,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]-resident",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Metasrv Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 267
- },
- "id": 271,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Metasrv CPU Usage per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current memory usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 0,
- "y": 277
- },
- "id": 272,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum(process_resident_memory_bytes{}) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "max(greptime_memory_limit_in_bytes{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Flownode Memory per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Current cpu usage by instance",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 10,
- "w": 12,
- "x": 12,
- "y": 277
- },
- "id": 273,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true,
- "sortBy": "Mean",
- "sortDesc": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{ pod }}]",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "exemplar": false,
- "expr": "max(greptime_cpu_limit_in_millicores{})",
- "hide": false,
- "instant": false,
- "legendFormat": "limit",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Flownode CPU Usage per Instance",
- "type": "timeseries"
- }
- ],
- "title": "Resources",
- "type": "row"
- },
- {
- "collapsed": true,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 24
- },
- "id": 280,
+ "id": 428,
"panels": [
{
"datasource": {
@@ -2312,11 +6561,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 288
+ "y": 123
},
- "id": 281,
+ "id": 470,
"options": {
"legend": {
"calcs": [
@@ -2355,7 +6604,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "HTTP P99 per Instance.",
+ "description": "HTTP P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2413,11 +6662,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 288
+ "w": 8,
+ "x": 8,
+ "y": 123
},
- "id": 282,
+ "id": 471,
"options": {
"legend": {
"calcs": [
@@ -2446,9 +6695,22 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~\"/health|/metrics\"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~\"/health|/metrics\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "HTTP P99 per Instance",
+ "title": "HTTP P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -2514,11 +6776,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 296
+ "w": 8,
+ "x": 16,
+ "y": 123
},
- "id": 283,
+ "id": 472,
"options": {
"legend": {
"calcs": [
@@ -2542,7 +6804,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]",
"range": true,
@@ -2557,7 +6819,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "gRPC P99 per Instance.",
+ "description": "gRPC P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2615,11 +6877,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 296
+ "w": 8,
+ "x": 0,
+ "y": 131
},
- "id": 284,
+ "id": 473,
"options": {
"legend": {
"calcs": [],
@@ -2641,14 +6903,27 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "gRPC P99 per Instance",
+ "title": "gRPC P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -2714,11 +6989,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 304
+ "w": 8,
+ "x": 8,
+ "y": 131
},
- "id": 285,
+ "id": 474,
"options": {
"legend": {
"calcs": [
@@ -2742,7 +7017,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))",
+ "expr": "sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]",
"range": true,
@@ -2757,7 +7032,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "MySQL P99 per Instance.",
+ "description": "MySQL P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -2815,11 +7090,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 304
+ "w": 8,
+ "x": 16,
+ "y": 131
},
- "id": 286,
+ "id": 475,
"options": {
"legend": {
"calcs": [
@@ -2844,14 +7119,27 @@
},
"editorMode": "code",
"exemplar": false,
- "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))",
"instant": false,
"legendFormat": "[{{ instance }}]-[{{ pod }}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "MySQL P99 per Instance",
+ "title": "MySQL P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -2917,11 +7205,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 312
+ "y": 139
},
- "id": 287,
+ "id": 476,
"options": {
"legend": {
"calcs": [
@@ -2945,7 +7233,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))",
+ "expr": "sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]",
"range": true,
@@ -2960,7 +7248,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "PostgreSQL P99 per Instance.",
+ "description": "PostgreSQL P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3018,11 +7306,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 312
+ "w": 8,
+ "x": 8,
+ "y": 139
},
- "id": 288,
+ "id": 477,
"options": {
"legend": {
"calcs": [],
@@ -3044,14 +7332,27 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-p99",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "PostgreSQL P99 per Instance",
+ "title": "PostgreSQL P99 and Avg per Instance",
"type": "timeseries"
}
],
@@ -3064,111 +7365,10 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 25
+ "y": 123
},
- "id": 289,
+ "id": 429,
"panels": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ingestion rate by row as in each frontend",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "rowsps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 6,
- "w": 24,
- "x": 0,
- "y": 289
- },
- "id": 292,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Ingest Rows per Instance",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -3232,11 +7432,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 295
+ "y": 124
},
- "id": 290,
+ "id": 479,
"options": {
"legend": {
"calcs": [
@@ -3260,7 +7460,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]",
"range": true,
@@ -3275,7 +7475,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Region Call P99 per Instance.",
+ "description": "Region Call P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3333,11 +7533,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 295
+ "w": 8,
+ "x": 8,
+ "y": 124
},
- "id": 291,
+ "id": 480,
"options": {
"legend": {
"calcs": [
@@ -3361,14 +7561,27 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval])))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Region Call P99 per Instance",
+ "title": "Region Call P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -3430,43 +7643,15 @@
},
"unit": "s"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[minipc-1:4000]-[]-[encode]-AVG",
- "[minipc-1:4000]-[]-[partition]-AVG",
- "[minipc-1:4000]-[]-[decode_request]-AVG",
- "[minipc-1:4000]-[]-[datanode_handle]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 303
+ "w": 8,
+ "x": 16,
+ "y": 124
},
- "id": 336,
+ "id": 481,
"options": {
"legend": {
"calcs": [
@@ -3510,22 +7695,662 @@
"refId": "B"
}
],
- "title": "Frontend Handle Bulk Insert Elapsed Time ",
+ "title": "Frontend Handle Bulk Insert Elapsed Time",
"type": "timeseries"
}
],
"title": "Frontend to Datanode",
"type": "row"
},
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 124
+ },
+ "id": 434,
+ "panels": [],
+ "title": "Datanode",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Datanode region request failures and failed inserts by instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 125,
+ "w": 8,
+ "h": 6
+ },
+ "id": 430,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "request-fail-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "failed-insert-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Region Request Failures and Failed Inserts",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito write rejects, write stall events, and active write stalling by datanode.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 125,
+ "w": 8,
+ "h": 6
+ },
+ "id": 431,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "reject-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "stall-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (greptime_mito_write_stalling_count)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "active-stalling-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Write Rejects and Stalls",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito flush and compaction failure rates by datanode.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "eps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 125,
+ "w": 8,
+ "h": 6
+ },
+ "id": 432,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "flush-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "compaction-[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flush and Compaction Failures",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 131,
+ "w": 8,
+ "h": 6
+ },
+ "id": 433,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_runs_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "runs",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "errors",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "files-deleted",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "orphaned-index-files",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "skipped-unparsable",
+ "range": true,
+ "refId": "E"
+ }
+ ],
+ "title": "Mito GC Health",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "P99 and average Mito garbage-collection duration by stage.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 131,
+ "w": 8,
+ "h": 6
+ },
+ "id": 551,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Mito GC Duration",
+ "type": "timeseries"
+ },
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 26
+ "y": 137
},
- "id": 293,
+ "id": 435,
"panels": [
{
"datasource": {
@@ -3590,11 +8415,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 193
+ "y": 138
},
- "id": 294,
+ "id": 482,
"options": {
"legend": {
"calcs": [
@@ -3618,7 +8443,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
"range": true,
@@ -3633,7 +8458,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Request P99 per Instance.",
+ "description": "Request P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -3691,11 +8516,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 193
+ "w": 8,
+ "x": 8,
+ "y": 138
},
- "id": 295,
+ "id": 483,
"options": {
"legend": {
"calcs": [
@@ -3719,14 +8544,155 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Request P99 per Instance",
+ "title": "Request P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 138
+ },
+ "id": 556,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{worker}}]-avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Request Wait P99 and Avg per Worker",
"type": "timeseries"
},
{
@@ -3792,11 +8758,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 225
+ "y": 146
},
- "id": 296,
+ "id": 484,
"options": {
"legend": {
"calcs": [
@@ -3820,7 +8786,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "greptime_mito_write_buffer_bytes{}",
+ "expr": "greptime_mito_write_buffer_bytes",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]",
"range": true,
@@ -3893,11 +8859,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 225
+ "w": 8,
+ "x": 8,
+ "y": 146
},
- "id": 297,
+ "id": 485,
"options": {
"legend": {
"calcs": [
@@ -3921,7 +8887,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]",
"range": true,
@@ -3931,207 +8897,6 @@
"title": "Write Rows per Instance",
"type": "timeseries"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Flush QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 233
- },
- "id": 298,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Flush OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stall per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 233
- },
- "id": 299,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_write_stall_total{})",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write Stall per Instance",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -4195,11 +8960,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 241
+ "w": 8,
+ "x": 16,
+ "y": 146
},
- "id": 300,
+ "id": 488,
"options": {
"legend": {
"calcs": [
@@ -4223,7 +8988,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage=\"total\"}[$__rate_interval]))",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage=\"total\"}[$__rate_interval]))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]",
"range": true,
@@ -4238,133 +9003,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Read Stage P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.87:4000]-[mycluster-datanode-1]-[cache_miss_read]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 241
- },
- "id": 301,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write Stage P99 per Instance.",
+ "description": "Read Stage P99 and average per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -4422,11 +9061,125 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 249
+ "y": 154
},
- "id": 302,
+ "id": 489,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Read Stage P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write Stage P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 154
+ },
+ "id": 490,
"options": {
"legend": {
"calcs": [
@@ -4452,213 +9205,11 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]",
"range": true,
"refId": "A"
- }
- ],
- "title": "Write Stage P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction OPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 249
- },
- "id": 303,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{ instance }}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction OPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction latency by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 257
- },
- "id": 304,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
- "range": true,
- "refId": "A"
},
{
"datasource": {
@@ -4666,7 +9217,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval]))",
"hide": false,
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
@@ -4674,235 +9225,7 @@
"refId": "B"
}
],
- "title": "Compaction Elapsed Time per Instance by Stage",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 257
- },
- "id": 305,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Compaction P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 265
- },
- "id": 306,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
- "range": true,
- "refId": "B"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
- "range": true,
- "refId": "C"
- }
- ],
- "title": "WAL write size",
+ "title": "Write Stage P99 and Avg per Instance",
"type": "timeseries"
},
{
@@ -4964,40 +9287,15 @@
},
"unit": "decbytes"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[10.39.145.79:4000]-[mycluster-datanode-0]-[file]"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 265
+ "w": 8,
+ "x": 16,
+ "y": 154
},
- "id": 307,
+ "id": 495,
"options": {
"legend": {
"calcs": [],
@@ -5019,7 +9317,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "greptime_mito_cache_bytes{}",
+ "expr": "greptime_mito_cache_bytes",
"instant": false,
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
"range": true,
@@ -5029,524 +9327,6 @@
"title": "Cached Bytes per Instance",
"type": "timeseries"
},
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing compaction task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 273
- },
- "id": 308,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_compaction_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Compaction",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Raft engine (local disk) log store sync latency, p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 273
- },
- "id": 310,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "WAL sync duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write-ahead log operations latency at p99",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 281
- },
- "id": 311,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Log Store op duration seconds",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Ongoing flush task count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 281
- },
- "id": 312,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_mito_inflight_flush_count",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Inflight Flush",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Compaction oinput output bytes",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "bytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 289
- },
- "id": 335,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_input_bytes)",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-input",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod) (greptime_mito_compaction_output_bytes)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-output",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Compaction Input/Output Bytes",
- "type": "timeseries"
- },
{
"datasource": {
"type": "prometheus",
@@ -5606,44 +9386,15 @@
},
"unit": "s"
},
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "[192.168.50.8:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[bulk_extend]-AVG",
- "[minipc-1:4000]-[]-[prepare_bulk_request]-AVG",
- "[minipc-1:4000]-[]-[process_bulk_req]-AVG",
- "[minipc-1:4000]-[]-[write_bulk]-AVG"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
- "legend": false,
- "tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
+ "overrides": []
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 289
+ "w": 8,
+ "x": 0,
+ "y": 162
},
- "id": 337,
+ "id": 501,
"options": {
"legend": {
"calcs": [
@@ -5695,7 +9446,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Compaction oinput output bytes",
+ "description": "Active series and field-builder counts per memtable by instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -5753,11 +9504,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 0,
- "y": 297
+ "w": 8,
+ "x": 8,
+ "y": 162
},
- "id": 348,
+ "id": 502,
"options": {
"legend": {
"calcs": [
@@ -5867,11 +9618,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 297
+ "w": 8,
+ "x": 16,
+ "y": 162
},
- "id": 338,
+ "id": 503,
"options": {
"legend": {
"calcs": [
@@ -5984,11 +9735,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 305
+ "y": 170
},
- "id": 349,
+ "id": 504,
"options": {
"legend": {
"calcs": [
@@ -6012,7 +9763,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))",
+ "expr": "sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval]))",
"legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]",
"range": true,
"refId": "A"
@@ -6022,7 +9773,7 @@
"type": "timeseries"
}
],
- "title": "Mito Engine",
+ "title": "Storage",
"type": "row"
},
{
@@ -6031,16 +9782,16 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 27
+ "y": 138
},
- "id": 313,
+ "id": 555,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "QPS per Instance.",
+ "description": "Flush QPS per Instance.",
"fieldConfig": {
"defaults": {
"color": {
@@ -6053,6 +9804,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -6096,12 +9848,12 @@
"overrides": []
},
"gridPos": {
- "h": 10,
- "w": 24,
+ "h": 8,
+ "w": 8,
"x": 0,
- "y": 459
+ "y": 139
},
- "id": 314,
+ "id": 486,
"options": {
"legend": {
"calcs": [
@@ -6117,7 +9869,7 @@
"sort": "desc"
}
},
- "pluginVersion": "12.0.0",
+ "pluginVersion": "11.6.0",
"targets": [
{
"datasource": {
@@ -6125,14 +9877,14 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))",
+ "expr": "sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval]))",
"instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{reason}}]",
"range": true,
"refId": "A"
}
],
- "title": "QPS per Instance",
+ "title": "Flush OPS per Instance",
"type": "timeseries"
},
{
@@ -6140,7 +9892,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Read QPS per Instance.",
+ "description": "Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs.",
"fieldConfig": {
"defaults": {
"color": {
@@ -6153,6 +9905,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -6191,823 +9944,21 @@
}
]
},
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 469
- },
- "id": 315,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~\"read|Reader::read\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read QPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Read P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
"unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 469
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 139
},
- "id": 316,
+ "id": 540,
"options": {
"legend": {
"calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~\"read|Reader::read\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Read P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 476
- },
- "id": 317,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write QPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 476
- },
- "id": 318,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Write P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "List QPS per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 483
- },
- "id": 319,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation=\"list\"}[$__rate_interval]))",
- "instant": false,
- "interval": "",
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "List QPS per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "List P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 483
- },
- "id": 320,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation=\"list\"}[$__rate_interval])))",
- "instant": false,
- "interval": "",
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "List P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Other Requests per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "ops"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 490
- },
- "id": 321,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|write|list|stat\"}[$__rate_interval]))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Other Requests per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Other Request P99 per Instance.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 3,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 490
- },
- "id": 322,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "multi",
- "sort": "desc"
- }
- },
- "pluginVersion": "12.0.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~\"read|write|list|Writer::write|Writer::close|Reader::read\"}[$__rate_interval])))",
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Other Request P99 per Instance",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Total traffic as in bytes by instance and operation",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "decbytes"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 7,
- "w": 12,
- "x": 0,
- "y": 497
- },
- "id": 323,
- "options": {
- "legend": {
- "calcs": [
- "lastNotNull"
+ "mean"
],
"displayMode": "table",
"placement": "bottom",
@@ -7019,26 +9970,49 @@
"sort": "none"
}
},
- "pluginVersion": "12.0.0",
+ "pluginVersion": "11.6.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "disableTextWrap": false,
"editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
+ "expr": "histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
"instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p95",
"range": true,
- "refId": "A",
- "useBackend": false
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{type}}]-avg",
+ "range": true,
+ "refId": "C"
}
],
- "title": "Opendal traffic",
+ "title": "Flush Elapsed Time",
"type": "timeseries"
},
{
@@ -7046,7 +10020,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "OpenDAL error counts per Instance.",
+ "description": "Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7059,6 +10033,7 @@
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
+ "barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
@@ -7096,17 +10071,247 @@
"value": 80
}
]
+ },
+ "unit": "Bps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*-files$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
}
+ ]
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 139
+ },
+ "id": 541,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-bytes",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-files",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Flush Throughput",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing flush task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
},
"overrides": []
},
"gridPos": {
- "h": 7,
- "w": 12,
- "x": 12,
- "y": 497
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 147
},
- "id": 334,
+ "id": 499,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_flush_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Flush",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction OPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 147
+ },
+ "id": 491,
"options": {
"legend": {
"calcs": [
@@ -7122,7 +10327,7 @@
"sort": "desc"
}
},
- "pluginVersion": "12.0.0",
+ "pluginVersion": "11.6.0",
"targets": [
{
"datasource": {
@@ -7130,18 +10335,461 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!=\"NotFound\"}[$__rate_interval]))",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))",
"instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]",
+ "legendFormat": "[{{ instance }}]-[{{pod}}]",
"range": true,
"refId": "A"
}
],
- "title": "OpenDAL errors per Instance",
+ "title": "Compaction OPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Ongoing compaction task count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 147
+ },
+ "id": 496,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_mito_inflight_compaction_count",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Inflight Compaction",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 155
+ },
+ "id": 493,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction latency by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 155
+ },
+ "id": 492,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{stage}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Elapsed Time per Instance by Stage",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "Bps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 155
+ },
+ "id": 500,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-input",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-output",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Compaction Input/Output Bytes",
"type": "timeseries"
}
],
- "title": "OpenDAL",
+ "title": "Flush and Compaction",
"type": "row"
},
{
@@ -7150,16 +10798,16 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 28
+ "y": 139
},
- "id": 351,
+ "id": 549,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Triggered region flush total",
+ "description": "Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7211,21 +10859,23 @@
}
]
},
- "unit": "none"
+ "unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 195
+ "y": 164,
+ "w": 8,
+ "h": 6
},
- "id": 350,
+ "id": 542,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7243,14 +10893,28 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "meta_triggered_region_flush_total",
+ "expr": "histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
+ "legendFormat": "{{type}}-p95",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-p99",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Triggered region flush total",
+ "title": "Index Apply Elapsed Time",
"type": "timeseries"
},
{
@@ -7258,7 +10922,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Triggered region checkpoint total",
+ "description": "Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7310,21 +10974,23 @@
}
]
},
- "unit": "none"
+ "unit": "s"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 195
+ "x": 8,
+ "y": 164,
+ "w": 8,
+ "h": 6
},
- "id": 352,
+ "id": 543,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7342,14 +11008,28 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "meta_triggered_region_checkpoint_total",
+ "expr": "histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
+ "legendFormat": "{{type}}-{{stage}}-p95",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-{{stage}}-p99",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Triggered region checkpoint total",
+ "title": "Index Create Elapsed Time",
"type": "timeseries"
},
{
@@ -7357,7 +11037,135 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Topic estimated max replay size",
+ "description": "Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "rowsps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*-bytes$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "Bps"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 164,
+ "w": 8,
+ "h": 6
+ },
+ "id": 544,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-rows",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-bytes",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Index Create Rows and Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7414,16 +11222,18 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 227
+ "y": 170,
+ "w": 8,
+ "h": 6
},
- "id": 353,
+ "id": 545,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7441,14 +11251,28 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "meta_topic_estimated_replay_size",
+ "expr": "greptime_index_apply_memory_usage",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{topic_name}}",
+ "legendFormat": "apply",
"range": true,
"refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (greptime_index_create_memory_usage)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "create-{{type}}",
+ "range": true,
+ "refId": "B"
}
],
- "title": "Topic estimated replay size",
+ "title": "Index Memory Usage",
"type": "timeseries"
},
{
@@ -7456,7 +11280,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Kafka logstore's bytes traffic",
+ "description": "Index read and write byte rates by operation and file type for puffin and intermediate files.",
"fieldConfig": {
"defaults": {
"color": {
@@ -7508,21 +11332,23 @@
}
]
},
- "unit": "bytes"
+ "unit": "Bps"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 227
+ "x": 8,
+ "y": 170,
+ "w": 8,
+ "h": 6
},
- "id": 354,
+ "id": 546,
"options": {
"legend": {
- "calcs": [],
- "displayMode": "list",
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
"placement": "bottom",
"showLegend": true
},
@@ -7540,18 +11366,249 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
+ "expr": "sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))",
+ "hide": false,
"instant": false,
- "legendFormat": "{{pod}}-{{logstore}}",
+ "legendFormat": "{{type}}-{{file_type}}",
"range": true,
"refId": "A"
}
],
- "title": "Kafka logstore's bytes traffic",
+ "title": "Index IO Bytes",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Index IO operation rates by operation and file type, including read, write, seek, and flush operations.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 170,
+ "w": 8,
+ "h": 6
+ },
+ "id": 547,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{type}}-{{file_type}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Index IO Operations",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Index-related cache hits, misses, and evictions from Mito caches.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 176,
+ "w": 8,
+ "h": 6
+ },
+ "id": 548,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_mito_cache_hit{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "hit-{{type}}",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type) (rate(greptime_mito_cache_miss{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "miss-{{type}}",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~\"index.*|vector_index|index_result\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "evict-{{type}}-{{cause}}",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Index Cache",
"type": "timeseries"
}
],
- "title": "Remote WAL",
+ "title": "Index",
"type": "row"
},
{
@@ -7560,10 +11617,292 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 29
+ "y": 140
},
- "id": 324,
+ "id": 451,
"panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 0
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 165,
+ "w": 8,
+ "h": 8
+ },
+ "id": 436,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(greptime_meta_inactive_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "inactive-regions",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(greptime_lease_expired_region)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "lease-expired",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Inactive and Lease-expired Regions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 165,
+ "w": 8,
+ "h": 8
+ },
+ "id": 437,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "meta-recv-rate",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(greptime_meta_heartbeat_connection_num)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "meta-connections",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "frontend-send",
+ "range": true,
+ "refId": "C"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "frontend-recv",
+ "range": true,
+ "refId": "D"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "datanode-send",
+ "range": true,
+ "refId": "E"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "datanode-recv",
+ "range": true,
+ "refId": "F"
+ }
+ ],
+ "title": "Heartbeat Health",
+ "type": "timeseries"
+ },
{
"datasource": {
"type": "prometheus",
@@ -7602,12 +11941,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 196
+ "x": 16,
+ "y": 165,
+ "w": 8,
+ "h": 8
},
- "id": 325,
+ "id": 525,
"options": {
"colWidth": 0.9,
"legend": {
@@ -7716,12 +12055,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 196
+ "x": 0,
+ "y": 173,
+ "w": 8,
+ "h": 8
},
- "id": 326,
+ "id": 526,
"options": {
"legend": {
"calcs": [],
@@ -7743,7 +12082,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "greptime_meta_region_migration_error",
+ "expr": "rate(greptime_meta_region_migration_error[$__rate_interval])",
"instant": false,
"legendFormat": "{{pod}}-{{state}}-{{error_type}}",
"range": true,
@@ -7815,12 +12154,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 228
+ "x": 8,
+ "y": 173,
+ "w": 8,
+ "h": 8
},
- "id": 327,
+ "id": 527,
"options": {
"legend": {
"calcs": [],
@@ -7914,12 +12253,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 228
+ "x": 16,
+ "y": 173,
+ "w": 8,
+ "h": 8
},
- "id": 339,
+ "id": 528,
"options": {
"legend": {
"calcs": [],
@@ -8013,12 +12352,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 236
+ "y": 181,
+ "w": 8,
+ "h": 8
},
- "id": 340,
+ "id": 529,
"options": {
"legend": {
"calcs": [],
@@ -8049,6 +12388,19 @@
"range": true,
"refId": "A",
"useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
"title": "SQL Execution Latency (RDS)",
@@ -8116,12 +12468,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 236
+ "x": 8,
+ "y": 181,
+ "w": 8,
+ "h": 8
},
- "id": 341,
+ "id": 530,
"options": {
"legend": {
"calcs": [],
@@ -8152,6 +12504,19 @@
"range": true,
"refId": "A",
"useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{pod}}]-[{{name}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
"title": "Handler Execution Latency",
@@ -8219,12 +12584,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 244
+ "x": 16,
+ "y": 181,
+ "w": 8,
+ "h": 8
},
- "id": 342,
+ "id": 531,
"options": {
"legend": {
"calcs": [],
@@ -8247,7 +12612,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))",
+ "expr": "histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
@@ -8265,7 +12630,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "Rate of heartbeats received by metasrv from datanodes and frontends.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8322,12 +12687,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 244
+ "x": 0,
+ "y": 189,
+ "w": 8,
+ "h": 8
},
- "id": 345,
+ "id": 532,
"options": {
"legend": {
"calcs": [],
@@ -8368,7 +12733,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "p99 and average latency of metasrv key-value store operations by op and target.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8425,12 +12790,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 252
+ "x": 8,
+ "y": 189,
+ "w": 8,
+ "h": 8
},
- "id": 343,
+ "id": 533,
"options": {
"legend": {
"calcs": [],
@@ -8453,7 +12818,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))",
+ "expr": "histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
@@ -8461,6 +12826,19 @@
"range": true,
"refId": "A",
"useBackend": false
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{pod}}]-[{{op}}]-[{{target}}]-avg",
+ "range": true,
+ "refId": "B"
}
],
"title": "Meta KV Ops Latency",
@@ -8471,7 +12849,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "Rate of metasrv key-value store operations by op.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8528,12 +12906,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 252
+ "x": 16,
+ "y": 189,
+ "w": 8,
+ "h": 8
},
- "id": 346,
+ "id": 534,
"options": {
"legend": {
"calcs": [],
@@ -8574,7 +12952,7 @@
"type": "prometheus",
"uid": "${metrics}"
},
- "description": "Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.",
+ "description": "p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step.",
"fieldConfig": {
"defaults": {
"color": {
@@ -8631,12 +13009,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
"x": 0,
- "y": 260
+ "y": 197,
+ "w": 8,
+ "h": 8
},
- "id": 347,
+ "id": 535,
"options": {
"legend": {
"calcs": [],
@@ -8659,7 +13037,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"includeNullMetadata": true,
"instant": false,
@@ -8675,7 +13053,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8692,7 +13070,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8709,7 +13087,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8726,7 +13104,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8743,7 +13121,7 @@
},
"disableTextWrap": false,
"editorMode": "code",
- "expr": "histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))",
+ "expr": "histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
@@ -8814,17 +13192,17 @@
}
]
},
- "unit": "s"
+ "unit": "ops"
},
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 260
+ "x": 8,
+ "y": 197,
+ "w": 8,
+ "h": 8
},
- "id": 355,
+ "id": 536,
"options": {
"legend": {
"calcs": [],
@@ -8846,7 +13224,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "greptime_meta_reconciliation_stats",
+ "expr": "rate(greptime_meta_reconciliation_stats[$__rate_interval])",
"hide": false,
"instant": false,
"legendFormat": "{{pod}}-{{table_type}}-{{type}}",
@@ -8919,12 +13297,12 @@
"overrides": []
},
"gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 268
+ "x": 16,
+ "y": 197,
+ "w": 8,
+ "h": 8
},
- "id": 356,
+ "id": 537,
"options": {
"legend": {
"calcs": [],
@@ -8946,7 +13324,7 @@
"uid": "${metrics}"
},
"editorMode": "code",
- "expr": "histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)",
+ "expr": "histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))",
"hide": false,
"instant": false,
"legendFormat": "{{procedure_name}}-{{step}}-P90",
@@ -8967,9 +13345,3174 @@
"h": 1,
"w": 24,
"x": 0,
- "y": 30
+ "y": 141
},
- "id": 328,
+ "id": 553,
+ "panels": [
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "custom": {
+ "align": "auto",
+ "cellOptions": {
+ "type": "auto",
+ "wrapText": false
+ },
+ "inspect": false
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byName",
+ "options": "disk_size"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "bytes"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 166,
+ "w": 24,
+ "h": 18
+ },
+ "id": 438,
+ "options": {
+ "cellHeight": "sm",
+ "footer": {
+ "countRows": false,
+ "enablePagination": true,
+ "fields": [
+ "region_rows",
+ "disk_size"
+ ],
+ "reducer": [
+ "sum"
+ ],
+ "show": true
+ },
+ "showHeader": true,
+ "sortBy": []
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Hotspot Regions",
+ "type": "table"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write load of each datanode over time.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 184,
+ "w": 8,
+ "h": 11
+ },
+ "id": 439,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "greptime_datanode_history_load",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write)",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Distribution of write load across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "min": 0,
+ "unit": "binBps"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 184,
+ "w": 8,
+ "h": 11
+ },
+ "id": 440,
+ "options": {
+ "displayLabels": [
+ "percent"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true
+ },
+ "pieType": "donut",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "exemplar": false,
+ "expr": "greptime_datanode_history_load",
+ "format": "time_series",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": true,
+ "legendFormat": "datanode-{{datanode_id}}({{instance}})",
+ "range": false,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Datanode Load(Write) Distribution",
+ "type": "piechart"
+ },
+ {
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "description": "Distribution of leader regions and data size across datanodes.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ }
+ },
+ "fieldMinMax": false,
+ "mappings": [],
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 184,
+ "w": 8,
+ "h": 11
+ },
+ "id": 441,
+ "options": {
+ "displayLabels": [
+ "value"
+ ],
+ "legend": {
+ "displayMode": "list",
+ "placement": "right",
+ "showLegend": true,
+ "values": [
+ "percent"
+ ]
+ },
+ "pieType": "pie",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "/^data_size$/",
+ "values": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "dataset": "information_schema",
+ "datasource": {
+ "type": "mysql",
+ "uid": "${information_schema}"
+ },
+ "editorMode": "code",
+ "format": "table",
+ "rawQuery": true,
+ "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;",
+ "refId": "A",
+ "sql": {
+ "columns": [
+ {
+ "parameters": [],
+ "type": "function"
+ }
+ ],
+ "groupBy": [
+ {
+ "property": {
+ "type": "string"
+ },
+ "type": "groupBy"
+ }
+ ],
+ "limit": 50
+ }
+ }
+ ],
+ "title": "Datanode Data Distribution",
+ "type": "piechart"
+ }
+ ],
+ "title": "Hotspot",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 142
+ },
+ "id": 554,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer action count",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 167,
+ "w": 8,
+ "h": 8
+ },
+ "id": 442,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 167,
+ "w": 8,
+ "h": 8
+ },
+ "id": 443,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 167,
+ "w": 8,
+ "h": 8
+ },
+ "id": 444,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_datanodes)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanodes",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer region count by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 175,
+ "w": 8,
+ "h": 8
+ },
+ "id": 445,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (state) (greptime_region_balancer_regions)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Regions",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Region balancer datanode stability statistics by state",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ }
+ ]
+ },
+ "unit": "binBps"
+ },
+ "overrides": [
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*count$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "short"
+ }
+ ]
+ },
+ {
+ "matcher": {
+ "id": "byRegexp",
+ "options": ".*ratio$"
+ },
+ "properties": [
+ {
+ "id": "unit",
+ "value": "percentunit"
+ }
+ ]
+ }
+ ]
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 175,
+ "w": 8,
+ "h": 8
+ },
+ "id": 446,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "auto",
+ "percentChangeColorMode": "standard",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showPercentChange": false,
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "builder",
+ "expr": "sum by (state) (greptime_region_balancer_datanode_stability)",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{state}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Region Balancer Datanode Stability",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition action count by result",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": -1,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineStyle": {
+ "fill": "solid"
+ },
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 175,
+ "w": 8,
+ "h": 8
+ },
+ "id": 447,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{result}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Actions",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition gate stop count by gate and reason",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 183,
+ "w": 8,
+ "h": 8
+ },
+ "id": 448,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{gate}} / {{reason}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Gate Stops",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition sampling elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 183,
+ "w": 8,
+ "h": 8
+ },
+ "id": 449,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Sampling P99",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Auto repartition executor elapsed time by stage",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "smooth",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 183,
+ "w": 8,
+ "h": 8
+ },
+ "id": 450,
+ "options": {
+ "legend": {
+ "calcs": [
+ "mean"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "{{stage}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Auto Repartition Executor P99",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Autopilot",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 143
+ },
+ "id": 452,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 10,
+ "w": 24,
+ "x": 0,
+ "y": 168
+ },
+ "id": 505,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 178
+ },
+ "id": 506,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"read|Reader::read\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Read QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Read P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 178
+ },
+ "id": 507,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~\"read|Reader::read\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~\"read|Reader::read\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"read|Reader::read\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Read P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 178
+ },
+ "id": 508,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Write QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 185
+ },
+ "id": 509,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ \"Writer::write|Writer::close|write\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~\"write|Writer::write|Writer::close\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Write P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "List QPS per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 185
+ },
+ "id": 510,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation=\"list\"}[$__rate_interval]))",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "List QPS per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "List P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 185
+ },
+ "id": 511,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation=\"list\"}[$__rate_interval])))",
+ "instant": false,
+ "interval": "",
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=\"list\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=\"list\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "List P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Other Requests per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 192
+ },
+ "id": 512,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Other Requests per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Other Request P99 and average per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 3,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 8,
+ "y": 192
+ },
+ "id": 513,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~\"read|Reader::read|write|Writer::write|Writer::close|list|stat\"}[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg",
+ "range": true,
+ "refId": "B"
+ }
+ ],
+ "title": "Other Request P99 and Avg per Instance",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Total traffic as in bytes by instance and operation",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "decbytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 16,
+ "y": 192
+ },
+ "id": 514,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "disableTextWrap": false,
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval]))",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Opendal traffic",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "OpenDAL error counts per Instance.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 7,
+ "w": 8,
+ "x": 0,
+ "y": 199
+ },
+ "id": 515,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "multi",
+ "sort": "desc"
+ }
+ },
+ "pluginVersion": "12.0.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!=\"NotFound\"}[$__rate_interval]))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "OpenDAL errors per Instance",
+ "type": "timeseries"
+ }
+ ],
+ "title": "Object Store",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 144
+ },
+ "id": 552,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 169
+ },
+ "id": 494,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p95",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-req-size-p99",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-throughput",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "WAL write size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Raft engine (local disk) log store sync latency, p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 169
+ },
+ "id": 497,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "WAL sync duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Write-ahead log operations latency at p99",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "points",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 169
+ },
+ "id": 498,
+ "options": {
+ "legend": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "displayMode": "table",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))",
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Log Store op duration seconds",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region flush total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 177
+ },
+ "id": 516,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_flush_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region flush total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Triggered region checkpoint total",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 8,
+ "y": 177
+ },
+ "id": 517,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_triggered_region_checkpoint_total",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Triggered region checkpoint total",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Topic estimated max replay size",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 16,
+ "y": 177
+ },
+ "id": 518,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "meta_topic_estimated_replay_size",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{topic_name}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Topic estimated replay size",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Kafka logstore's bytes traffic",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 8,
+ "x": 0,
+ "y": 185
+ },
+ "id": 519,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])",
+ "instant": false,
+ "legendFormat": "{{pod}}-{{logstore}}",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Kafka logstore's bytes traffic",
+ "type": "timeseries"
+ }
+ ],
+ "title": "WAL",
+ "type": "row"
+ },
+ {
+ "collapsed": true,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 145
+ },
+ "id": 453,
"panels": [
{
"datasource": {
@@ -9032,11 +16575,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
+ "w": 8,
"x": 0,
- "y": 3282
+ "y": 170
},
- "id": 329,
+ "id": 520,
"options": {
"legend": {
"calcs": [],
@@ -9127,11 +16670,11 @@
},
"gridPos": {
"h": 8,
- "w": 12,
- "x": 12,
- "y": 3282
+ "w": 8,
+ "x": 8,
+ "y": 170
},
- "id": 330,
+ "id": 521,
"options": {
"legend": {
"calcs": [],
@@ -9169,6 +16712,19 @@
"legendFormat": "[{{instance}}]-[{{pod}}]-p99",
"range": true,
"refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "[{{instance}}]-[{{pod}}]-avg",
+ "range": true,
+ "refId": "C"
}
],
"title": "Flow Ingest Latency",
@@ -9235,11 +16791,11 @@
},
"gridPos": {
"h": 8,
- "w": 9,
- "x": 0,
- "y": 3290
+ "w": 8,
+ "x": 16,
+ "y": 170
},
- "id": 331,
+ "id": 522,
"options": {
"legend": {
"calcs": [],
@@ -9343,11 +16899,11 @@
},
"gridPos": {
"h": 8,
- "w": 9,
- "x": 9,
- "y": 3290
+ "w": 8,
+ "x": 0,
+ "y": 178
},
- "id": 332,
+ "id": 523,
"options": {
"legend": {
"calcs": [],
@@ -9438,11 +16994,11 @@
},
"gridPos": {
"h": 8,
- "w": 6,
- "x": 18,
- "y": 3290
+ "w": 8,
+ "x": 8,
+ "y": 178
},
- "id": 333,
+ "id": 524,
"options": {
"legend": {
"calcs": [],
@@ -9477,2077 +17033,793 @@
"type": "row"
},
{
- "collapsed": false,
+ "collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
- "y": 31
+ "y": 146
},
- "id": 357,
- "panels": [],
- "title": "Trigger",
- "type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "bf9fzta69bhtsa"
- },
- "description": "Total number of triggers currently defined.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
+ "id": 461,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
},
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
+ "description": "Total number of triggers currently defined.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
},
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 32
- },
- "id": 358,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "greptime_trigger_count{}",
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Trigger Count",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 40
- },
- "id": 359,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-p75",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Trigger Eval Elapsed",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Rate of failed trigger evaluations.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 40
- },
- "id": 360,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Trigger Eval Failure Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed time to send trigger alerts to notification channels.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 48
- },
- "id": 361,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Send Alert Elapsed",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Rate of failures when sending trigger alerts.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 48
- },
- "id": 364,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Send Alert Failure Rate",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Elapsed time to persist trigger alert records.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 56
- },
- "id": 363,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99",
- "range": true,
- "refId": "A"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75",
- "range": true,
- "refId": "B"
- }
- ],
- "title": "Save Alert Elapsed",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Rate of failures when persisting trigger alert records.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "none"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 56
- },
- "id": 362,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])",
- "hide": false,
- "instant": false,
- "legendFormat": "__auto",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Save Alert Failure Rate",
- "type": "timeseries"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 64
- },
- "id": 366,
- "panels": [],
- "title": "Hotspot",
- "type": "row"
- },
- {
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "custom": {
- "align": "auto",
- "cellOptions": {
- "type": "auto",
- "wrapText": false
- },
- "inspect": false
- },
- "fieldMinMax": false,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- }
- },
- "overrides": [
- {
- "matcher": {
- "id": "byName",
- "options": "disk_size"
- },
- "properties": [
- {
- "id": "unit",
- "value": "bytes"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 18,
- "w": 24,
- "x": 0,
- "y": 65
- },
- "id": 365,
- "options": {
- "cellHeight": "sm",
- "footer": {
- "countRows": false,
- "enablePagination": true,
- "fields": [
- "region_rows",
- "disk_size"
- ],
- "reducer": [
- "sum"
- ],
- "show": true
- },
- "showHeader": true,
- "sortBy": []
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "dataset": "information_schema",
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "WITH table_stats AS (\n SELECT\n table_id,\n COUNT(*) AS region_count,\n SUM(disk_size) AS total_disk_size,\n SUM(region_rows) as total_region_rows\n FROM information_schema.region_statistics\n WHERE region_role = 'Leader'\n GROUP BY table_id\n HAVING COUNT(*) > 1\n)\n\nSELECT\n t.table_schema,\n t.table_name,\n\n r.region_id,\n t.table_id,\n r.region_number,\n\n p.partition_description,\n\n\n ROUND(\n r.disk_size * 100.0\n / NULLIF(ts.total_disk_size, 0),\n 2\n ) AS disk_size_share_percent,\n\n r.disk_size,\n \n ROUND(\n r.region_rows * 100.0\n / NULLIF(ts.total_region_rows, 0),\n 2\n ) AS region_rows_share_percent,\n r.region_rows\n\nFROM information_schema.region_statistics r\n\nJOIN table_stats ts\n ON r.table_id = ts.table_id\n\nJOIN information_schema.tables t\n ON r.table_id = t.table_id\n\nLEFT JOIN information_schema.partitions p\n ON p.table_schema = t.table_schema\n AND p.table_name = t.table_name\n AND p.greptime_partition_id = r.region_id\n\nWHERE r.region_role = 'Leader'\n\nORDER BY region_rows_share_percent DESC\nLIMIT 100;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
- },
- "type": "groupBy"
- }
- ],
- "limit": 50
- }
- }
- ],
- "title": "Hotspot Regions",
- "type": "table"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Write load of each datanode over time.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "total_region_rows",
- "total_written_bytes_since_open",
- "total_disk_size",
- "total_sst_size",
- "total_sst_num",
- "total_memtable_size",
- "total_manifest_size",
- "total_index_size",
- "cluster_disk_ratio_percent",
- "cluster_sst_ratio_percent",
- "cluster_rows_ratio_percent",
- "max_region_disk_size",
- "max_region_rows",
- "leader_region_count",
- "disk_ratio_percent",
- "data_size"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": []
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 0,
- "y": 83
- },
- "id": 368,
- "options": {
- "legend": {
- "calcs": [],
- "displayMode": "list",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "expr": "greptime_datanode_history_load",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": false,
- "legendFormat": "datanode-{{datanode_id}}({{instance}})",
- "range": true,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Datanode Load(Write)",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Distribution of write load across datanodes.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "min": 0,
- "unit": "binBps"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 12,
- "y": 83
- },
- "id": 369,
- "options": {
- "displayLabels": [
- "percent"
- ],
- "legend": {
- "displayMode": "list",
- "placement": "right",
- "showLegend": true
- },
- "pieType": "donut",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "disableTextWrap": false,
- "editorMode": "builder",
- "exemplar": false,
- "expr": "greptime_datanode_history_load",
- "format": "time_series",
- "fullMetaSearch": false,
- "includeNullMetadata": true,
- "instant": true,
- "legendFormat": "datanode-{{datanode_id}}({{instance}})",
- "range": false,
- "refId": "A",
- "useBackend": false
- }
- ],
- "title": "Datanode Load(Write) Distribution",
- "type": "piechart"
- },
- {
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "description": "Distribution of leader regions and data size across datanodes.",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- }
- },
- "fieldMinMax": false,
- "mappings": [],
- "unit": "bytes"
- },
- "overrides": [
- {
- "__systemRef": "hideSeriesFrom",
- "matcher": {
- "id": "byNames",
- "options": {
- "mode": "exclude",
- "names": [
- "total_region_rows",
- "total_written_bytes_since_open",
- "total_disk_size",
- "total_sst_size",
- "total_sst_num",
- "total_memtable_size",
- "total_manifest_size",
- "total_index_size",
- "cluster_disk_ratio_percent",
- "cluster_sst_ratio_percent",
- "cluster_rows_ratio_percent",
- "max_region_disk_size",
- "max_region_rows",
- "leader_region_count",
- "disk_ratio_percent",
- "data_size"
- ],
- "prefix": "All except:",
- "readOnly": true
- }
- },
- "properties": [
- {
- "id": "custom.hideFrom",
- "value": {
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
"legend": false,
"tooltip": false,
- "viz": true
- }
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 11,
- "w": 12,
- "x": 0,
- "y": 94
- },
- "id": 367,
- "options": {
- "displayLabels": [
- "value"
- ],
- "legend": {
- "displayMode": "list",
- "placement": "right",
- "showLegend": true,
- "values": [
- "percent"
- ]
- },
- "pieType": "pie",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "/^data_size$/",
- "values": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "dataset": "information_schema",
- "datasource": {
- "type": "mysql",
- "uid": "${information_schema}"
- },
- "editorMode": "code",
- "format": "table",
- "rawQuery": true,
- "rawSql": "WITH leader_regions AS (\n SELECT\n CONCAT(\n 'datanode-',\n p.peer_id,\n ' (',\n p.peer_addr,\n ')'\n ) AS datanode,\n r.disk_size\n FROM information_schema.region_statistics r\n JOIN information_schema.region_peers p\n ON r.region_id = p.region_id\n WHERE r.region_role = 'Leader'\n AND p.is_leader = 'Yes'\n)\n\nSELECT\n datanode,\n COUNT(*) AS leader_region_count,\n SUM(disk_size) AS data_size\nFROM leader_regions\nGROUP BY datanode\nORDER BY data_size DESC;",
- "refId": "A",
- "sql": {
- "columns": [
- {
- "parameters": [],
- "type": "function"
- }
- ],
- "groupBy": [
- {
- "property": {
- "type": "string"
+ "viz": false
},
- "type": "groupBy"
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
}
- ],
- "limit": 50
- }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 171,
+ "w": 8,
+ "h": 6
+ },
+ "id": 454,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "greptime_trigger_count",
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Trigger Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed time for trigger evaluation, including query execution and condition evaluation.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 171,
+ "w": 8,
+ "h": 6
+ },
+ "id": 455,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p75",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Trigger Eval Elapsed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Rate of failed trigger evaluations.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 171,
+ "w": 8,
+ "h": 6
+ },
+ "id": 456,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_trigger_evaluate_failure_count[$__rate_interval])",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Trigger Eval Failure Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed time to send trigger alerts to notification channels.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 177,
+ "w": 8,
+ "h": 6
+ },
+ "id": 457,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p75",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Send Alert Elapsed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Rate of failures when sending trigger alerts.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 8,
+ "y": 177,
+ "w": 8,
+ "h": 6
+ },
+ "id": 458,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_trigger_send_alert_failure_count[$__rate_interval])",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Send Alert Failure Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Elapsed time to persist trigger alert records.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 16,
+ "y": 177,
+ "w": 8,
+ "h": 6
+ },
+ "id": 459,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p99",
+ "range": true,
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "p75",
+ "range": true,
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "avg",
+ "range": true,
+ "refId": "C"
+ }
+ ],
+ "title": "Save Alert Elapsed",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "description": "Rate of failures when persisting trigger alert records.",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "barWidthFactor": 0.6,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green"
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 183,
+ "w": 8,
+ "h": 6
+ },
+ "id": 460,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "hideZeros": false,
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "pluginVersion": "11.6.0",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "${metrics}"
+ },
+ "editorMode": "code",
+ "expr": "rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])",
+ "hide": false,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A"
+ }
+ ],
+ "title": "Save Alert Failure Rate",
+ "type": "timeseries"
}
],
- "title": "Datanode Data Distribution",
- "type": "piechart"
- },
- {
- "collapsed": false,
- "gridPos": {
- "h": 1,
- "w": 24,
- "x": 0,
- "y": 105
- },
- "id": 370,
- "panels": [],
- "title": "Autopilot",
+ "title": "Trigger",
"type": "row"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer action count",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 106
- },
- "id": 371,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (result) (\n changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{result}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Actions",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer gate stop count by gate and reason",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 106
- },
- "id": 372,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{gate}} / {{reason}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Gate Stops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer datanode count by state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 114
- },
- "id": 373,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (state) (greptime_region_balancer_datanodes)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{state}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Datanodes",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer region count by state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 114
- },
- "id": 374,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (state) (greptime_region_balancer_regions)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{state}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Regions",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Region balancer datanode stability statistics by state",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "thresholds"
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- }
- ]
- },
- "unit": "binBps"
- },
- "overrides": [
- {
- "matcher": {
- "id": "byRegexp",
- "options": ".*count$"
- },
- "properties": [
- {
- "id": "unit",
- "value": "short"
- }
- ]
- },
- {
- "matcher": {
- "id": "byRegexp",
- "options": ".*ratio$"
- },
- "properties": [
- {
- "id": "unit",
- "value": "percentunit"
- }
- ]
- }
- ]
- },
- "gridPos": {
- "h": 8,
- "w": 24,
- "x": 0,
- "y": 122
- },
- "id": 375,
- "options": {
- "colorMode": "value",
- "graphMode": "area",
- "justifyMode": "auto",
- "orientation": "auto",
- "percentChangeColorMode": "standard",
- "reduceOptions": {
- "calcs": [
- "lastNotNull"
- ],
- "fields": "",
- "values": false
- },
- "showPercentChange": false,
- "textMode": "auto",
- "wideLayout": true
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "builder",
- "expr": "sum by (state) (greptime_region_balancer_datanode_stability)",
- "hide": false,
- "instant": false,
- "legendFormat": "{{state}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Region Balancer Datanode Stability",
- "type": "stat"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition action count by result",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": -1,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineStyle": {
- "fill": "solid"
- },
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 130
- },
- "id": 376,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{result}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Actions",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition gate stop count by gate and reason",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "line",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "short"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 130
- },
- "id": 377,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{gate}} / {{reason}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Gate Stops",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition sampling elapsed time by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "linear",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 0,
- "y": 138
- },
- "id": 378,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{stage}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Sampling P99",
- "type": "timeseries"
- },
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "description": "Auto repartition executor elapsed time by stage",
- "fieldConfig": {
- "defaults": {
- "color": {
- "mode": "palette-classic"
- },
- "custom": {
- "axisBorderShow": false,
- "axisCenteredZero": false,
- "axisColorMode": "text",
- "axisLabel": "",
- "axisPlacement": "auto",
- "barAlignment": 0,
- "barWidthFactor": 0.6,
- "drawStyle": "points",
- "fillOpacity": 0,
- "gradientMode": "none",
- "hideFrom": {
- "legend": false,
- "tooltip": false,
- "viz": false
- },
- "insertNulls": false,
- "lineInterpolation": "smooth",
- "lineWidth": 1,
- "pointSize": 5,
- "scaleDistribution": {
- "type": "linear"
- },
- "showPoints": "auto",
- "spanNulls": false,
- "stacking": {
- "group": "A",
- "mode": "none"
- },
- "thresholdsStyle": {
- "mode": "off"
- }
- },
- "mappings": [],
- "thresholds": {
- "mode": "absolute",
- "steps": [
- {
- "color": "green"
- },
- {
- "color": "red",
- "value": 80
- }
- ]
- },
- "unit": "s"
- },
- "overrides": []
- },
- "gridPos": {
- "h": 8,
- "w": 12,
- "x": 12,
- "y": 138
- },
- "id": 379,
- "options": {
- "legend": {
- "calcs": [
- "mean"
- ],
- "displayMode": "table",
- "placement": "bottom",
- "showLegend": true
- },
- "tooltip": {
- "hideZeros": false,
- "mode": "single",
- "sort": "none"
- }
- },
- "pluginVersion": "11.6.0",
- "targets": [
- {
- "datasource": {
- "type": "prometheus",
- "uid": "${metrics}"
- },
- "editorMode": "code",
- "expr": "histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))",
- "hide": false,
- "instant": false,
- "legendFormat": "{{stage}}",
- "range": true,
- "refId": "A"
- }
- ],
- "title": "Auto Repartition Executor P99",
- "type": "timeseries"
}
],
"preload": false,
@@ -11686,7 +17958,7 @@
},
"timepicker": {},
"timezone": "",
- "title": "GreptimeDB Copy",
+ "title": "GreptimeDB",
"uid": "dflfbxbwvvchsa",
- "version": 67
-}
\ No newline at end of file
+ "version": 68
+}
diff --git a/grafana/dashboards/metrics/standalone/dashboard.md b/grafana/dashboards/metrics/standalone/dashboard.md
index 672a2a8364..a3a894cffa 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.md
+++ b/grafana/dashboards/metrics/standalone/dashboard.md
@@ -4,144 +4,154 @@
| Uptime | `time() - process_start_time_seconds` | `stat` | The start time of GreptimeDB. | `prometheus` | `s` | `__auto` |
| Version | `SELECT pkg_version FROM information_schema.build_info` | `stat` | GreptimeDB version. | `mysql` | -- | -- |
| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `stat` | Total ingestion rate. | `prometheus` | `rowsps` | `__auto` |
-| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
-| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
+| Total Query Rate | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `stat` | Total query API call rate across MySQL, PostgreSQL, and PromQL frontends. | `prometheus` | `reqps` | `queries` |
+| User-facing Error Rate | `sum(rate(greptime_servers_error[$__rate_interval]))` | `stat` | Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures. | `prometheus` | `eps` | `errors` |
+| Recent Restarts | `sum(changes(process_start_time_seconds[$__range]))` | `stat` | Process restarts over the selected time range across GreptimeDB roles. | `prometheus` | `short` | `restarts` |
| Deployment | `SELECT count(*) as datanode FROM information_schema.cluster_info WHERE peer_type = 'DATANODE';`
`SELECT count(*) as frontend FROM information_schema.cluster_info WHERE peer_type = 'FRONTEND';`
`SELECT count(*) as metasrv FROM information_schema.cluster_info WHERE peer_type = 'METASRV';`
`SELECT count(*) as flownode FROM information_schema.cluster_info WHERE peer_type = 'FLOWNODE';` | `stat` | The deployment topology of GreptimeDB. | `mysql` | -- | -- |
| Database Resources | `SELECT COUNT(*) as databases FROM information_schema.schemata WHERE schema_name NOT IN ('greptime_private', 'information_schema')`
`SELECT COUNT(*) as tables FROM information_schema.tables WHERE table_schema != 'information_schema'`
`SELECT COUNT(region_id) as regions FROM information_schema.region_peers`
`SELECT COUNT(*) as flows FROM information_schema.flows` | `stat` | The number of the key resources in GreptimeDB. | `mysql` | -- | -- |
+| Total Storage Size | `select SUM(disk_size) from information_schema.region_statistics;` | `stat` | Total number of data file size. | `mysql` | `decbytes` | -- |
+| Total Rows | `select SUM(region_rows) from information_schema.region_statistics;` | `stat` | Total number of data rows in the cluster. Calculated by sum of rows from each region. | `mysql` | `sishort` | -- |
| Data Size | `SELECT SUM(memtable_size) * 0.42825 as WAL FROM information_schema.region_statistics;`
`SELECT SUM(index_size) as index FROM information_schema.region_statistics;`
`SELECT SUM(manifest_size) as manifest FROM information_schema.region_statistics;` | `stat` | The data size of wal/index/manifest in the GreptimeDB. | `mysql` | `decbytes` | -- |
+| Total Ingestion Rate Trend | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `timeseries` | Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row. | `prometheus` | `rowsps` | `ingestion` |
+| Total Query Rate Trend | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row. | `prometheus` | `reqps` | `queries` |
+| HTTP Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))`
`sum(rate(greptime_servers_http_requests_elapsed_sum{path!~"/health\|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints. | `prometheus` | `s` | `http-p99` |
+| gRPC Request P99 and Avg | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | Tail and average latency for gRPC requests served by frontends. | `prometheus` | `s` | `grpc-p99` |
# Ingestion
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Total ingestion rate.
Here we listed 3 primary protocols:
- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` |
-| Ingestion Rate by Type | `sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))` | `timeseries` | Total ingestion rate.
Here we listed 3 primary protocols:
- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `http-logs` |
-# Queries
+| Total Ingestion Rate | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))` | `timeseries` | Total ingestion rate.
Here we listed 3 primary protocols:
- Prometheus remote write
- Greptime's gRPC API (when using our ingest SDK)
- Log ingestion http API
| `prometheus` | `rowsps` | `ingestion` |
+| Ingestion Rate by Protocol | `sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))`
`sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval]))`
`sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval]))` | `timeseries` | Rows, samples, or documents ingested by primary observability and table-ingestion protocols. | `prometheus` | `rowsps` | `table-operator` |
+| Ingestion Latency by Protocol | `histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints. | `prometheus` | `s` | `prometheus-write` |
+| Bulk Insert Message Rows and Size | `sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval]))`
`sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval]))` | `timeseries` | Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency. | `prometheus` | `rowsps` | `rows` |
+| Prom Store Flush Pipeline | `sum(rate(greptime_prom_store_flush_total[$__rate_interval]))`
`sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval]))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval])))` | `timeseries` | Remote-write pending-row flush operations, flushed rows, and p99 flush latency. | `prometheus` | `short` | `flush-ops` |
+| OTLP Trace Failures | `sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval]))` | `timeseries` | OTLP trace ingestion failures reported by frontends. | `prometheus` | `eps` | `trace-failures` |
+# Health
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Total Query Rate | `sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))`
`sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))` | `timeseries` | Total rate of query API calls by protocol. This metric is collected from frontends.
Here we listed 3 main protocols:
- MySQL
- Postgres
- Prometheus API
Note that there are some other minor query APIs like /sql are not included | `prometheus` | `reqps` | `mysql` |
+| Protocol Error Rates | `sum by (protocol) (rate(greptime_servers_error[$__rate_interval]))`
`sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval]))`
`sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics",code!~"2.."}[$__rate_interval]))`
`sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~"0\|OK"}[$__rate_interval]))` | `timeseries` | User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related. | `prometheus` | `eps` | `server-{{protocol}}` |
+| Frontend and Query Rejections | `sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval]))`
`sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))` | `timeseries` | Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution. | `prometheus` | `rps` | `request-memory` |
+| Datanode Write Failures | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))` | `timeseries` | Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing. | `prometheus` | `eps` | `region-request-[{{instance}}]-[{{pod}}]` |
+| Buffered Ingestion Loss | `sum(rate(greptime_pending_rows_flush_failures[$__rate_interval]))`
`sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval]))` | `timeseries` | Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal. | `prometheus` | `eps` | `flush-failures` |
+| Mito Backpressure and Failures | `sum(rate(greptime_mito_write_reject_total[$__rate_interval]))`
`sum(rate(greptime_mito_write_stall_total[$__rate_interval]))`
`sum(rate(greptime_mito_flush_failure_total[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_failure_total[$__rate_interval]))` | `timeseries` | Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes. | `prometheus` | `eps` | `write-reject` |
+| Scan and Compaction Memory Rejects | `sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))`
`sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))`
`sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))` | `timeseries` | Datanode scan and compaction memory rejection/exhaustion counters. | `prometheus` | `rps` | `scan-rejected` |
+| OpenDAL Errors | `sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | Object-store errors by scheme, operation, and error, excluding NotFound noise. | `prometheus` | `eps` | `{{scheme}}-{{operation}}-{{error}}` |
+| Metasrv Failures | `sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))`
`sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))` | `timeseries` | Region migration and reconciliation failures in metasrv. | `prometheus` | `eps` | `migration-fail` |
+| Flow and Trigger Failures | `sum by (code) (rate(greptime_flow_errors[$__rate_interval]))`
`sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))`
`sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))` | `timeseries` | Derived-data and alerting pipeline failures. | `prometheus` | `eps` | `flow-{{code}}` |
+| Mito GC Failures | `sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection errors and skipped/orphaned files on datanodes. | `prometheus` | `short` | `gc-errors` |
+# Capacity
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Runtime Threads | `sum by (instance, pod) (greptime_runtime_threads_alive)`
`sum by (instance, pod) (greptime_runtime_threads_idle)` | `timeseries` | Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation. | `prometheus` | `short` | `alive-[{{instance}}]-[{{pod}}]` |
+| Request Memory Utilization | `sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes)` | `timeseries` | Frontend request memory usage divided by configured request memory limit. | `prometheus` | `percentunit` | `[{{instance}}]-[{{pod}}]` |
+| Query Memory Usage | `sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)` | `timeseries` | Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]` |
+| Scan and Compaction Memory | `sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes)`
`sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes)`
`sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes)` | `timeseries` | Datanode scan memory usage and compaction memory utilization. | `prometheus` | `bytes` | `scan-[{{instance}}]-[{{pod}}]` |
+| Write Buffer and Active Stalling | `sum by (instance, pod) (greptime_mito_write_buffer_bytes)`
`sum by (instance, pod) (greptime_mito_write_stalling_count)` | `timeseries` | Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure. | `prometheus` | `bytes` | `buffer-[{{instance}}]-[{{pod}}]` |
+| Prom Store Backlog | `sum by (instance, pod) (greptime_prom_store_pending_rows)`
`sum by (instance, pod) (greptime_prom_store_pending_batches)`
`sum by (instance, pod) (greptime_prom_store_pending_workers)` | `timeseries` | Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog. | `prometheus` | `short` | `rows-[{{instance}}]-[{{pod}}]` |
+| Inflight Flush and Compaction | `sum by (instance, pod) (greptime_mito_inflight_flush_count)`
`sum by (instance, pod) (greptime_mito_inflight_compaction_count)` | `timeseries` | Current in-flight flush and compaction tasks on datanodes. | `prometheus` | `short` | `flush-[{{instance}}]-[{{pod}}]` |
# Resources
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Datanode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
-| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
-| Frontend Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
-| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` |
-| Metasrv Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
-| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
-| Flownode Memory per Instance | `sum(process_resident_memory_bytes{}) by (instance, pod)`
`max(greptime_memory_limit_in_bytes{})` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
-| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores{})` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
+| Frontend CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]-cpu` |
+| Datanode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
+| Metasrv CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
+| Frontend Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
+| Datanode Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{instance}}]-[{{ pod }}]` |
+| Metasrv Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]-resident` |
+| Flownode CPU Usage per Instance | `sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)`
`max(greptime_cpu_limit_in_millicores)` | `timeseries` | Current cpu usage by instance | `prometheus` | `none` | `[{{ instance }}]-[{{ pod }}]` |
+| Flownode Memory per Instance | `sum(process_resident_memory_bytes) by (instance, pod)`
`max(greptime_memory_limit_in_bytes)` | `timeseries` | Current memory usage by instance | `prometheus` | `bytes` | `[{{ instance }}]-[{{ pod }}]` |
+# Queries
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Query Rate by Protocol | `sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | Query API call rates by protocol, collected from frontends. | `prometheus` | `reqps` | `mysql` |
+| Query Latency by Protocol | `histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))`
`sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))` | `timeseries` | p95, p99, and average query latency by main frontend protocol. | `prometheus` | `s` | `mysql-p95` |
+| Query Stage Latency | `histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))` | `timeseries` | p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks. | `prometheus` | `s` | `p95-{{stage}}` |
+| Merge Scan Fan-out and Errors | `sum by (instance, pod) (greptime_query_merge_scan_regions)`
`sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))` | `timeseries` | Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans. | `prometheus` | `short` | `regions-[{{instance}}]-[{{pod}}]` |
+| Pushdown Fallback Errors | `sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))` | `timeseries` | Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work. | `prometheus` | `eps` | `pushdown-fallback-errors` |
+| PromQL Series Count | `sum by (instance, pod) (greptime_promql_series_count)` | `timeseries` | Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness. | `prometheus` | `short` | `[{{instance}}]-[{{pod}}]` |
+| Connections and Prepared Statements | `sum by (instance, pod) (greptime_servers_mysql_connection_count)`
`sum by (instance, pod) (greptime_servers_postgres_connection_count)`
`sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval]))` | `timeseries` | MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks. | `prometheus` | `short` | `mysql-connections-[{{instance}}]-[{{pod}}]` |
# Frontend Requests
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
| HTTP QPS per Instance | `sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]` |
-| HTTP P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))` | `timeseries` | HTTP P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
-| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` |
-| gRPC P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | gRPC P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
-| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
-| MySQL P99 per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | MySQL P99 per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
-| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
-| PostgreSQL P99 per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | PostgreSQL P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
+| HTTP P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health\|/metrics"}[$__rate_interval])))`
`sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~"/health\|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health\|/metrics"}[$__rate_interval]))` | `timeseries` | HTTP P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
+| gRPC QPS per Instance | `sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | gRPC QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]` |
+| gRPC P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))` | `timeseries` | gRPC P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99` |
+| MySQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))` | `timeseries` | MySQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
+| MySQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))` | `timeseries` | MySQL P99 and average per Instance. | `prometheus` | `s` | `[{{ instance }}]-[{{ pod }}]-p99` |
+| PostgreSQL QPS per Instance | `sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))` | `timeseries` | PostgreSQL QPS per Instance. | `prometheus` | `reqps` | `[{{instance}}]-[{{pod}}]` |
+| PostgreSQL P99 and Avg per Instance | `histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))`
`sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))` | `timeseries` | PostgreSQL P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
# Frontend to Datanode
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Ingest Rows per Instance | `sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))` | `timeseries` | Ingestion rate by row as in each frontend | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
-| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
-| Region Call P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))` | `timeseries` | Region Call P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
-| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
-# Mito Engine
+| Region Call QPS per Instance | `sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))` | `timeseries` | Region Call QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
+| Region Call P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval])))`
`sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))` | `timeseries` | Region Call P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{request_type}}]` |
+| Frontend Handle Bulk Insert Elapsed Time | `sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_table_operator_handle_bulk_insert_count[$__rate_interval]))`
`histogram_quantile(0.99, sum by(instance, pod, stage, le) (rate(greptime_table_operator_handle_bulk_insert_bucket[$__rate_interval])))` | `timeseries` | Per-stage time for frontend to handle bulk insert requests | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG` |
+# Datanode
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-| Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-| Write Buffer per Instance | `greptime_mito_write_buffer_bytes{}` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` |
-| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
-| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
-| Write Stall per Instance | `sum by(instance, pod) (greptime_mito_write_stall_total{})` | `timeseries` | Write Stall per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
-| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` |
-| Read Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Read Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
-| Write Stage P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Write Stage P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
-| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
-| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
-| Compaction P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))` | `timeseries` | Compaction P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction` |
-| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
-| Cached Bytes per Instance | `greptime_mito_cache_bytes{}` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
-| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
-| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
-| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
-| Compaction Input/Output Bytes | `sum by(instance, pod) (greptime_mito_compaction_input_bytes)`
`sum by(instance, pod) (greptime_mito_compaction_output_bytes)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-input` |
+| Region Request Failures and Failed Inserts | `sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))` | `timeseries` | Datanode region request failures and failed inserts by instance. | `prometheus` | `eps` | `request-fail-[{{instance}}]-[{{pod}}]` |
+| Write Rejects and Stalls | `sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval]))`
`sum by (instance, pod) (greptime_mito_write_stalling_count)` | `timeseries` | Mito write rejects, write stall events, and active write stalling by datanode. | `prometheus` | `short` | `reject-[{{instance}}]-[{{pod}}]` |
+| Flush and Compaction Failures | `sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval]))` | `timeseries` | Mito flush and compaction failure rates by datanode. | `prometheus` | `eps` | `flush-[{{instance}}]-[{{pod}}]` |
+| Mito GC Health | `sum(rate(greptime_mito_gc_runs_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval]))`
`sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))`
`sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))` | `timeseries` | Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files. | `prometheus` | `short` | `runs` |
+| Mito GC Duration | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval])))`
`sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval]))` | `timeseries` | P99 and average Mito garbage-collection duration by stage. | `prometheus` | `s` | `{{stage}}-p99` |
+# Storage
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Request OPS per Instance | `sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))` | `timeseries` | Request QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
+| Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))` | `timeseries` | Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
+| Request Wait P99 and Avg per Worker | `histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))`
`sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval]))` | `timeseries` | Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{worker}}]-p95` |
+| Write Buffer per Instance | `greptime_mito_write_buffer_bytes` | `timeseries` | Write Buffer per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]` |
+| Write Rows per Instance | `sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval]))` | `timeseries` | Ingestion size by row counts. | `prometheus` | `rowsps` | `[{{instance}}]-[{{pod}}]` |
+| Read Stage OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage="total"}[$__rate_interval]))` | `timeseries` | Read Stage OPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]` |
+| Read Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Read Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
+| Write Stage P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Write Stage P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]` |
+| Cached Bytes per Instance | `greptime_mito_cache_bytes` | `timeseries` | Cached Bytes per Instance. | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
| Region Worker Handle Bulk Insert Requests | `histogram_quantile(0.95, sum by(le,instance, stage, pod) (rate(greptime_region_worker_handle_write_bucket[$__rate_interval])))`
`sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_sum[$__rate_interval]))/sum by(instance, stage, pod) (rate(greptime_region_worker_handle_write_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to handle bulk insert region requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
-| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Compaction oinput output bytes | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
+| Active Series and Field Builders Count | `sum by(instance, pod) (greptime_mito_memtable_active_series_count)`
`sum by(instance, pod) (greptime_mito_memtable_field_builder_count)` | `timeseries` | Active series and field-builder counts per memtable by instance. | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]-series` |
| Region Worker Convert Requests | `histogram_quantile(0.95, sum by(le, instance, stage, pod) (rate(greptime_datanode_convert_region_request_bucket[$__rate_interval])))`
`sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_sum[$__rate_interval]))/sum by(le,instance, stage, pod) (rate(greptime_datanode_convert_region_request_count[$__rate_interval]))` | `timeseries` | Per-stage elapsed time for region worker to decode requests. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-P95` |
-| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
-# OpenDAL
+| Cache Miss | `sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval]))` | `timeseries` | The local cache miss of the datanode. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]` |
+# Flush and Compaction
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Read P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))` | `timeseries` | Read P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Write P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))` | `timeseries` | Write P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
-| List P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))` | `timeseries` | List P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
-| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|write\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Other Request P99 per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read\|write\|list\|Writer::write\|Writer::close\|Reader::read"}[$__rate_interval])))` | `timeseries` | Other Request P99 per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
-| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
-# Remote WAL
+| Flush OPS per Instance | `sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval]))` | `timeseries` | Flush QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{reason}}]` |
+| Flush Elapsed Time | `histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))`
`sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))` | `timeseries` | Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
+| Flush Throughput | `sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))`
`sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))` | `timeseries` | Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-bytes` |
+| Inflight Flush | `greptime_mito_inflight_flush_count` | `timeseries` | Ongoing flush task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
+| Compaction OPS per Instance | `sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction OPS per Instance. | `prometheus` | `ops` | `[{{ instance }}]-[{{pod}}]` |
+| Inflight Compaction | `greptime_mito_inflight_compaction_count` | `timeseries` | Ongoing compaction task count | `prometheus` | `none` | `[{{instance}}]-[{{pod}}]` |
+| Compaction P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
+| Compaction Elapsed Time per Instance by Stage | `histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))`
`sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))` | `timeseries` | Compaction latency by stage | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{stage}}]-p99` |
+| Compaction Input/Output Bytes | `sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))`
`sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))` | `timeseries` | Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume. | `prometheus` | `Bps` | `[{{instance}}]-[{{pod}}]-input` |
+# Index
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
-| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
-| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
-| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
-| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
+| Index Apply Elapsed Time | `histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates. | `prometheus` | `s` | `{{type}}-p95` |
+| Index Create Elapsed Time | `histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))` | `timeseries` | Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays. | `prometheus` | `s` | `{{type}}-{{stage}}-p95` |
+| Index Create Rows and Bytes | `sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))`
`sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))` | `timeseries` | Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure. | `prometheus` | `rowsps` | `{{type}}-rows` |
+| Index Memory Usage | `greptime_index_apply_memory_usage`
`sum by (type) (greptime_index_create_memory_usage)` | `timeseries` | Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work. | `prometheus` | `bytes` | `apply` |
+| Index IO Bytes | `sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))` | `timeseries` | Index read and write byte rates by operation and file type for puffin and intermediate files. | `prometheus` | `Bps` | `{{type}}-{{file_type}}` |
+| Index IO Operations | `sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))` | `timeseries` | Index IO operation rates by operation and file type, including read, write, seek, and flush operations. | `prometheus` | `ops` | `{{type}}-{{file_type}}` |
+| Index Cache | `sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))`
`sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*\|vector_index\|index_result"}[$__rate_interval]))` | `timeseries` | Index-related cache hits, misses, and evictions from Mito caches. | `prometheus` | `ops` | `hit-{{type}}` |
# Metasrv
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
+| Inactive and Lease-expired Regions | `sum(greptime_meta_inactive_regions)`
`sum(greptime_lease_expired_region)` | `timeseries` | Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues. | `prometheus` | `short` | `inactive-regions` |
+| Heartbeat Health | `sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))`
`sum(greptime_meta_heartbeat_connection_num)`
`sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval]))`
`sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval]))`
`sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval]))` | `timeseries` | Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters. | `prometheus` | `short` | `meta-recv-rate` |
| Region migration datanode | `greptime_meta_region_migration_stat{datanode_type="src"}`
`greptime_meta_region_migration_stat{datanode_type="desc"}` | `status-history` | Counter of region migration by source and destination | `prometheus` | -- | `from-datanode-{{datanode_id}}` |
-| Region migration error | `greptime_meta_region_migration_error` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
+| Region migration error | `rate(greptime_meta_region_migration_error[$__rate_interval])` | `timeseries` | Counter of region migration error | `prometheus` | `none` | `{{pod}}-{{state}}-{{error_type}}` |
| Datanode load | `greptime_datanode_load` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `binBps` | `Datanode-{{datanode_id}}-writeload` |
| Rate of SQL Executions (RDS) | `rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval])` | `timeseries` | Displays the rate of SQL executions processed by the Meta service using the RDS backend. | `prometheus` | `none` | `{{pod}} {{op}} {{type}} {{result}} ` |
-| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
+| SQL Execution Latency (RDS) | `histogram_quantile(0.90, sum by(pod, op, type, result, le) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_bucket[$__rate_interval])))`
`sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))` | `timeseries` | Measures the response time of SQL executions via the RDS backend. | `prometheus` | `ms` | `{{pod}} {{op}} {{type}} {{result}} p90` |
| Handler Execution Latency | `histogram_quantile(0.90, sum by(pod, le, name) (
rate(greptime_meta_handler_execute_bucket[$__rate_interval])
-))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` |
-| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` |
-| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}` |
-| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
-| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
-| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))`
`histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))` | `timeseries` | Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
-| Reconciliation stats | `greptime_meta_reconciliation_stats` | `timeseries` | Reconciliation stats | `prometheus` | `s` | `{{pod}}-{{table_type}}-{{type}}` |
-| Reconciliation steps | `histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
-# Flownode
-| Title | Query | Type | Description | Datasource | Unit | Legend Format |
-| --- | --- | --- | --- | --- | --- | --- |
-| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
-| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
-| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
-| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
-| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
-# Trigger
-| Title | Query | Type | Description | Datasource | Unit | Legend Format |
-| --- | --- | --- | --- | --- | --- | --- |
-| Trigger Count | `greptime_trigger_count{}` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
-| Trigger Eval Elapsed | `histogram_quantile(0.99,
- rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
-)`
`histogram_quantile(0.75,
- rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])
-)` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
-| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
-| Send Alert Elapsed | `histogram_quantile(0.99,
- rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
-)`
`histogram_quantile(0.75,
- rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])
-)` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99` |
-| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
-| Save Alert Elapsed | `histogram_quantile(0.99,
- rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
-)`
`histogram_quantile(0.75,
- rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])
-)` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99` |
-| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
+))`
`sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))` | `timeseries` | Shows latency of Meta handlers by pod and handler name, useful for monitoring handler performance and detecting latency spikes.
| `prometheus` | `s` | `{{pod}} {{name}} p90` |
+| Heartbeat Packet Size | `histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))` | `timeseries` | Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
| `prometheus` | `bytes` | `{{pod}}` |
+| Meta Heartbeat Receive Rate | `rate(greptime_meta_heartbeat_rate[$__rate_interval])` | `timeseries` | Rate of heartbeats received by metasrv from datanodes and frontends. | `prometheus` | `s` | `{{pod}}` |
+| Meta KV Ops Latency | `histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))`
`sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))` | `timeseries` | p99 and average latency of metasrv key-value store operations by op and target. | `prometheus` | `s` | `{{pod}}-{{op}} p99` |
+| Rate of meta KV Ops | `rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])` | `timeseries` | Rate of metasrv key-value store operations by op. | `prometheus` | `none` | `{{pod}}-{{op}} p99` |
+| DDL Latency | `histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))`
`histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))` | `timeseries` | p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step. | `prometheus` | `s` | `CreateLogicalTables-{{step}} p90` |
+| Reconciliation stats | `rate(greptime_meta_reconciliation_stats[$__rate_interval])` | `timeseries` | Reconciliation stats | `prometheus` | `ops` | `{{pod}}-{{table_type}}-{{type}}` |
+| Reconciliation steps | `histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))` | `timeseries` | Elapsed of Reconciliation steps | `prometheus` | `s` | `{{procedure_name}}-{{step}}-P90` |
# Hotspot
| Title | Query | Type | Description | Datasource | Unit | Legend Format |
| --- | --- | --- | --- | --- | --- | --- |
@@ -240,3 +250,45 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
| Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
| Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
+# Object Store
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval]))` | `timeseries` | QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Read QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Read P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read\|Reader::read"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"read\|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read\|Reader::read"}[$__rate_interval]))` | `timeseries` | Read P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Write QPS per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Write P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ "Writer::write\|Writer::close\|write"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write\|Writer::write\|Writer::close"}[$__rate_interval]))` | `timeseries` | Write P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| List QPS per Instance | `sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))` | `timeseries` | List QPS per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
+| List P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="list"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))` | `timeseries` | List P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]` |
+| Other Requests per Instance | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Requests per Instance. | `prometheus` | `ops` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Other Request P99 and Avg per Instance | `histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])))`
`sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read\|Reader::read\|write\|Writer::write\|Writer::close\|list\|stat"}[$__rate_interval]))` | `timeseries` | Other Request P99 and average per Instance. | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| Opendal traffic | `sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval]))` | `timeseries` | Total traffic as in bytes by instance and operation | `prometheus` | `decbytes` | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]` |
+| OpenDAL errors per Instance | `sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))` | `timeseries` | OpenDAL error counts per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]` |
+# WAL
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| WAL write size | `histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))`
`sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))` | `timeseries` | Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate. | `prometheus` | `bytes` | `[{{instance}}]-[{{pod}}]-req-size-p95` |
+| WAL sync duration seconds | `histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))` | `timeseries` | Raft engine (local disk) log store sync latency, p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-p99` |
+| Log Store op duration seconds | `histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))` | `timeseries` | Write-ahead log operations latency at p99 | `prometheus` | `s` | `[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99` |
+| Triggered region flush total | `meta_triggered_region_flush_total` | `timeseries` | Triggered region flush total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
+| Triggered region checkpoint total | `meta_triggered_region_checkpoint_total` | `timeseries` | Triggered region checkpoint total | `prometheus` | `none` | `{{pod}}-{{topic_name}}` |
+| Topic estimated replay size | `meta_topic_estimated_replay_size` | `timeseries` | Topic estimated max replay size | `prometheus` | `bytes` | `{{pod}}-{{topic_name}}` |
+| Kafka logstore's bytes traffic | `rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])` | `timeseries` | Kafka logstore's bytes traffic | `prometheus` | `bytes` | `{{pod}}-{{logstore}}` |
+# Flownode
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Flow Ingest / Output Rate | `sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))` | `timeseries` | Flow Ingest / Output Rate. | `prometheus` | -- | `[{{pod}}]-[{{instance}}]-[{{direction}}]` |
+| Flow Ingest Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))`
`sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))` | `timeseries` | Flow Ingest Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-p95` |
+| Flow Operation Latency | `histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))`
`histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))` | `timeseries` | Flow Operation Latency. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{type}}]-p95` |
+| Flow Buffer Size per Instance | `greptime_flow_input_buf_size` | `timeseries` | Flow Buffer Size per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]` |
+| Flow Processing Error per Instance | `sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))` | `timeseries` | Flow Processing Error per Instance. | `prometheus` | -- | `[{{instance}}]-[{{pod}}]-[{{code}}]` |
+# Trigger
+| Title | Query | Type | Description | Datasource | Unit | Legend Format |
+| --- | --- | --- | --- | --- | --- | --- |
+| Trigger Count | `greptime_trigger_count` | `timeseries` | Total number of triggers currently defined. | `prometheus` | -- | `__auto` |
+| Trigger Eval Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time for trigger evaluation, including query execution and condition evaluation. | `prometheus` | `s` | `p99` |
+| Trigger Eval Failure Rate | `rate(greptime_trigger_evaluate_failure_count[$__rate_interval])` | `timeseries` | Rate of failed trigger evaluations. | `prometheus` | `none` | `__auto` |
+| Send Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to send trigger alerts to notification channels. | `prometheus` | `s` | `p99` |
+| Send Alert Failure Rate | `rate(greptime_trigger_send_alert_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when sending trigger alerts. | `prometheus` | `none` | `__auto` |
+| Save Alert Elapsed | `histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))`
`sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))` | `timeseries` | Elapsed time to persist trigger alert records. | `prometheus` | `s` | `p99` |
+| Save Alert Failure Rate | `rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])` | `timeseries` | Rate of failures when persisting trigger alert records. | `prometheus` | `none` | `__auto` |
diff --git a/grafana/dashboards/metrics/standalone/dashboard.yaml b/grafana/dashboards/metrics/standalone/dashboard.yaml
index c23501c074..0ebad934ba 100644
--- a/grafana/dashboards/metrics/standalone/dashboard.yaml
+++ b/grafana/dashboards/metrics/standalone/dashboard.yaml
@@ -29,24 +29,36 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: __auto
- - title: Total Storage Size
+ - title: Total Query Rate
type: stat
- description: Total number of data file size.
- unit: decbytes
+ description: Total query API call rate across MySQL, PostgreSQL, and PromQL frontends.
+ unit: reqps
queries:
- - expr: select SUM(disk_size) from information_schema.region_statistics;
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))
datasource:
- type: mysql
- uid: ${information_schema}
- - title: Total Rows
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: queries
+ - title: User-facing Error Rate
type: stat
- description: Total number of data rows in the cluster. Calculated by sum of rows from each region.
- unit: sishort
+ description: Server protocol errors returned by frontends. Sustained non-zero values indicate user-visible failures.
+ unit: eps
queries:
- - expr: select SUM(region_rows) from information_schema.region_statistics;
+ - expr: sum(rate(greptime_servers_error[$__rate_interval]))
datasource:
- type: mysql
- uid: ${information_schema}
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: errors
+ - title: Recent Restarts
+ type: stat
+ description: Process restarts over the selected time range across GreptimeDB roles.
+ unit: short
+ queries:
+ - expr: sum(changes(process_start_time_seconds[$__range]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: restarts
- title: Deployment
type: stat
description: The deployment topology of GreptimeDB.
@@ -87,6 +99,24 @@ groups:
datasource:
type: mysql
uid: ${information_schema}
+ - title: Total Storage Size
+ type: stat
+ description: Total number of data file size.
+ unit: decbytes
+ queries:
+ - expr: select SUM(disk_size) from information_schema.region_statistics;
+ datasource:
+ type: mysql
+ uid: ${information_schema}
+ - title: Total Rows
+ type: stat
+ description: Total number of data rows in the cluster. Calculated by sum of rows from each region.
+ unit: sishort
+ queries:
+ - expr: select SUM(region_rows) from information_schema.region_statistics;
+ datasource:
+ type: mysql
+ uid: ${information_schema}
- title: Data Size
type: stat
description: The data size of wal/index/manifest in the GreptimeDB.
@@ -104,6 +134,56 @@ groups:
datasource:
type: mysql
uid: ${information_schema}
+ - title: Total Ingestion Rate Trend
+ type: timeseries
+ description: Total ingestion throughput trend across frontends. Protocol breakdown is in the Ingestion row.
+ unit: rowsps
+ queries:
+ - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: ingestion
+ - title: Total Query Rate Trend
+ type: timeseries
+ description: Total query API call rate trend across frontend protocols. Protocol breakdown is in the Queries row.
+ unit: reqps
+ queries:
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval])) + sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval])) + sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: queries
+ - title: HTTP Request P99 and Avg
+ type: timeseries
+ description: Tail and average latency for HTTP requests served by frontends. Excludes health and metrics endpoints.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health|/metrics"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-p99
+ - expr: sum(rate(greptime_servers_http_requests_elapsed_sum{path!~"/health|/metrics"}[$__rate_interval])) / sum(rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-avg
+ - title: gRPC Request P99 and Avg
+ type: timeseries
+ description: Tail and average latency for gRPC requests served by frontends.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-p99
+ - expr: sum(rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-avg
- title: Ingestion
panels:
- title: Total Ingestion Rate
@@ -118,76 +198,483 @@ groups:
- Log ingestion http API
unit: rowsps
queries:
- - expr: sum(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))
+ - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: ingestion
- - title: Ingestion Rate by Type
+ - title: Ingestion Rate by Protocol
type: timeseries
- description: |
- Total ingestion rate.
-
- Here we listed 3 primary protocols:
-
- - Prometheus remote write
- - Greptime's gRPC API (when using our ingest SDK)
- - Log ingestion http API
+ description: Rows, samples, or documents ingested by primary observability and table-ingestion protocols.
unit: rowsps
queries:
- - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))
+ - expr: sum(rate(greptime_table_operator_ingest_rows[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: http-logs
+ legendFormat: table-operator
- expr: sum(rate(greptime_servers_prometheus_remote_write_samples[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: prometheus-remote-write
- - title: Queries
- panels:
- - title: Total Query Rate
+ - expr: sum(rate(greptime_servers_http_logs_ingestion_counter[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-logs
+ - expr: sum(rate(greptime_servers_loki_logs_ingestion_counter[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: loki-logs
+ - expr: sum(rate(greptime_servers_elasticsearch_logs_docs_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: elasticsearch-docs
+ - expr: sum(rate(greptime_frontend_otlp_metrics_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-metrics
+ - expr: sum(rate(greptime_frontend_otlp_logs_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-logs
+ - expr: sum(rate(greptime_frontend_otlp_traces_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-traces
+ - title: Ingestion Latency by Protocol
type: timeseries
- description: |-
- Total rate of query API calls by protocol. This metric is collected from frontends.
-
- Here we listed 3 main protocols:
- - MySQL
- - Postgres
- - Prometheus API
-
- Note that there are some other minor query APIs like /sql are not included
- unit: reqps
+ description: p99 and average HTTP ingestion latency for Prometheus remote write, logs, Loki, Elasticsearch, and OTLP endpoints.
+ unit: s
queries:
- - expr: sum (rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_prometheus_write_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: mysql
- - expr: sum (rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))
+ legendFormat: prometheus-write
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_logs_ingestion_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: pg
- - expr: sum (rate(greptime_servers_http_promql_elapsed_counte{}[$__rate_interval]))
+ legendFormat: http-logs
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_loki_logs_ingestion_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: promql
- - title: Resources
- panels:
- - title: Datanode Memory per Instance
+ legendFormat: loki-logs
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: elasticsearch
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_metrics_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-metrics
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_logs_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-logs
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_otlp_traces_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-traces
+ - expr: sum(rate(greptime_servers_http_prometheus_write_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_prometheus_write_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: prom-write-avg
+ - expr: sum(rate(greptime_servers_http_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_logs_ingestion_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-logs-avg
+ - expr: sum(rate(greptime_servers_loki_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_loki_logs_ingestion_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: loki-logs-avg
+ - expr: sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_elasticsearch_logs_ingestion_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: elasticsearch-avg
+ - expr: sum(rate(greptime_servers_http_otlp_metrics_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_metrics_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-metrics-avg
+ - expr: sum(rate(greptime_servers_http_otlp_logs_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_logs_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-logs-avg
+ - expr: sum(rate(greptime_servers_http_otlp_traces_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_otlp_traces_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: otlp-traces-avg
+ - title: Bulk Insert Message Rows and Size
type: timeseries
- description: Current memory usage by instance
+ description: Bulk-insert message row and byte rates. Spikes here can explain frontend bulk-insert latency.
+ unit: rowsps
+ queries:
+ - expr: sum(rate(greptime_table_operator_bulk_insert_message_rows_sum[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: rows
+ - expr: sum(rate(greptime_table_operator_bulk_insert_message_size_sum[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: bytes
+ - title: Prom Store Flush Pipeline
+ type: timeseries
+ description: Remote-write pending-row flush operations, flushed rows, and p99 flush latency.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_prom_store_flush_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-ops
+ - expr: sum(rate(greptime_prom_store_flush_rows_sum[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-rows
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_prom_store_flush_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-p99
+ - title: OTLP Trace Failures
+ type: timeseries
+ description: OTLP trace ingestion failures reported by frontends.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_frontend_otlp_traces_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trace-failures
+ - title: Health
+ panels:
+ - title: Protocol Error Rates
+ type: timeseries
+ description: User-facing and protocol-level error rates. Use labels to identify whether failures are server, auth, HTTP, or gRPC related.
+ unit: eps
+ queries:
+ - expr: sum by (protocol) (rate(greptime_servers_error[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: server-{{protocol}}
+ - expr: sum by (code) (rate(greptime_servers_auth_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: auth-{{code}}
+ - expr: sum by (path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics",code!~"2.."}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-{{method}}-{{path}}-{{code}}
+ - expr: sum by (path, code) (rate(greptime_servers_grpc_requests_elapsed_count{code!~"0|OK"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-{{path}}-{{code}}
+ - title: Frontend and Query Rejections
+ type: timeseries
+ description: Request and query memory rejections. Non-zero values indicate requests are being rejected before or during execution.
+ unit: rps
+ queries:
+ - expr: sum(rate(greptime_servers_request_memory_rejected_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: request-memory
+ - expr: sum(rate(greptime_query_memory_pool_rejected_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: query-memory
+ - title: Datanode Write Failures
+ type: timeseries
+ description: Region request failures and failed inserts on datanodes. These indicate backend write-path errors after routing.
+ unit: eps
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: region-request-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: failed-insert-[{{instance}}]-[{{pod}}]
+ - title: Buffered Ingestion Loss
+ type: timeseries
+ description: Pending-row flush failures and dropped rows. Sustained non-zero dropped rows are a data-loss signal.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_pending_rows_flush_failures[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-failures
+ - expr: sum(rate(greptime_pending_rows_flush_dropped_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: dropped-rows
+ - title: Mito Backpressure and Failures
+ type: timeseries
+ description: Storage-engine write rejects, write stalls, flush failures, and compaction failures on datanodes.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_mito_write_reject_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: write-reject
+ - expr: sum(rate(greptime_mito_write_stall_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: write-stall
+ - expr: sum(rate(greptime_mito_flush_failure_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-failure
+ - expr: sum(rate(greptime_mito_compaction_failure_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-failure
+ - title: Scan and Compaction Memory Rejects
+ type: timeseries
+ description: Datanode scan and compaction memory rejection/exhaustion counters.
+ unit: rps
+ queries:
+ - expr: sum(rate(greptime_mito_scan_requests_rejected_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: scan-rejected
+ - expr: sum(rate(greptime_mito_scan_memory_exhausted_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: scan-exhausted
+ - expr: sum(rate(greptime_mito_compaction_memory_rejected_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-rejected
+ - title: OpenDAL Errors
+ type: timeseries
+ description: Object-store errors by scheme, operation, and error, excluding NotFound noise.
+ unit: eps
+ queries:
+ - expr: sum by (scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{scheme}}-{{operation}}-{{error}}'
+ - title: Metasrv Failures
+ type: timeseries
+ description: Region migration and reconciliation failures in metasrv.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_meta_region_migration_fail[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: migration-fail
+ - expr: sum(rate(greptime_meta_reconciliation_procedure_error[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: reconciliation-error
+ - title: Flow and Trigger Failures
+ type: timeseries
+ description: Derived-data and alerting pipeline failures.
+ unit: eps
+ queries:
+ - expr: sum by (code) (rate(greptime_flow_errors[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flow-{{code}}
+ - expr: sum(rate(greptime_trigger_evaluate_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trigger-evaluate
+ - expr: sum(rate(greptime_trigger_send_alert_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trigger-send-alert
+ - expr: sum(rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: trigger-save-alert
+ - title: Mito GC Failures
+ type: timeseries
+ description: Mito garbage-collection errors and skipped/orphaned files on datanodes.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: gc-errors
+ - expr: sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: orphaned-index-files
+ - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: skipped-unparsable
+ - title: Capacity
+ panels:
+ - title: Runtime Threads
+ type: timeseries
+ description: Runtime thread pool size and idle threads by instance. Low idle threads during latency spikes can indicate executor saturation.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_runtime_threads_alive)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: alive-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_runtime_threads_idle)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: idle-[{{instance}}]-[{{pod}}]
+ - title: Request Memory Utilization
+ type: timeseries
+ description: Frontend request memory usage divided by configured request memory limit.
+ unit: percentunit
+ queries:
+ - expr: sum by (instance, pod) (greptime_servers_request_memory_in_use_bytes) / sum by (instance, pod) (greptime_servers_request_memory_limit_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Query Memory Usage
+ type: timeseries
+ description: Query memory pool usage. Use this with query memory rejection panels to diagnose query saturation.
unit: bytes
queries:
- - expr: sum(process_resident_memory_bytes{}) by (instance, pod)
+ - expr: sum by (instance, pod) (greptime_query_memory_pool_usage_bytes)
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{ pod }}]'
- - expr: max(greptime_memory_limit_in_bytes{})
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Scan and Compaction Memory
+ type: timeseries
+ description: Datanode scan memory usage and compaction memory utilization.
+ unit: bytes
+ queries:
+ - expr: sum by (instance, pod) (greptime_mito_scan_memory_usage_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: scan-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_compaction_memory_in_use_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-used-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_compaction_memory_limit_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-limit-[{{instance}}]-[{{pod}}]
+ - title: Write Buffer and Active Stalling
+ type: timeseries
+ description: Mito write buffer bytes and active write-stalling gauges. Growth here indicates write-path backpressure.
+ unit: bytes
+ queries:
+ - expr: sum by (instance, pod) (greptime_mito_write_buffer_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: buffer-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_write_stalling_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: stalling-[{{instance}}]-[{{pod}}]
+ - title: Prom Store Backlog
+ type: timeseries
+ description: Prometheus remote-write pending rows, batches, and workers. Rising pending rows indicate remote-write buffering backlog.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_prom_store_pending_rows)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: rows-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_prom_store_pending_batches)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: batches-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_prom_store_pending_workers)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: workers-[{{instance}}]-[{{pod}}]
+ - title: Inflight Flush and Compaction
+ type: timeseries
+ description: Current in-flight flush and compaction tasks on datanodes.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_mito_inflight_flush_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_inflight_compaction_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-[{{instance}}]-[{{pod}}]
+ - title: Resources
+ panels:
+ - title: Frontend CPU Usage per Instance
+ type: timeseries
+ description: Current cpu usage by instance
+ unit: none
+ queries:
+ - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu'
+ - expr: max(greptime_cpu_limit_in_millicores)
datasource:
type: prometheus
uid: ${metrics}
@@ -197,57 +684,12 @@ groups:
description: Current cpu usage by instance
unit: none
queries:
- - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)
+ - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_cpu_limit_in_millicores{})
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: limit
- - title: Frontend Memory per Instance
- type: timeseries
- description: Current memory usage by instance
- unit: bytes
- queries:
- - expr: sum(process_resident_memory_bytes{}) by (instance, pod)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_memory_limit_in_bytes{})
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: limit
- - title: Frontend CPU Usage per Instance
- type: timeseries
- description: Current cpu usage by instance
- unit: none
- queries:
- - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{ pod }}]-cpu'
- - expr: max(greptime_cpu_limit_in_millicores{})
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: limit
- - title: Metasrv Memory per Instance
- type: timeseries
- description: Current memory usage by instance
- unit: bytes
- queries:
- - expr: sum(process_resident_memory_bytes{}) by (instance, pod)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{ pod }}]-resident'
- - expr: max(greptime_memory_limit_in_bytes{})
+ - expr: max(greptime_cpu_limit_in_millicores)
datasource:
type: prometheus
uid: ${metrics}
@@ -257,27 +699,57 @@ groups:
description: Current cpu usage by instance
unit: none
queries:
- - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)
+ - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_cpu_limit_in_millicores{})
+ - expr: max(greptime_cpu_limit_in_millicores)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: limit
- - title: Flownode Memory per Instance
+ - title: Frontend Memory per Instance
type: timeseries
description: Current memory usage by instance
unit: bytes
queries:
- - expr: sum(process_resident_memory_bytes{}) by (instance, pod)
+ - expr: sum(process_resident_memory_bytes) by (instance, pod)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_memory_limit_in_bytes{})
+ - expr: max(greptime_memory_limit_in_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: limit
+ - title: Datanode Memory per Instance
+ type: timeseries
+ description: Current memory usage by instance
+ unit: bytes
+ queries:
+ - expr: sum(process_resident_memory_bytes) by (instance, pod)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{ pod }}]'
+ - expr: max(greptime_memory_limit_in_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: limit
+ - title: Metasrv Memory per Instance
+ type: timeseries
+ description: Current memory usage by instance
+ unit: bytes
+ queries:
+ - expr: sum(process_resident_memory_bytes) by (instance, pod)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{ pod }}]-resident'
+ - expr: max(greptime_memory_limit_in_bytes)
datasource:
type: prometheus
uid: ${metrics}
@@ -287,16 +759,193 @@ groups:
description: Current cpu usage by instance
unit: none
queries:
- - expr: sum(rate(process_cpu_seconds_total{}[$__rate_interval]) * 1000) by (instance, pod)
+ - expr: sum(rate(process_cpu_seconds_total[$__rate_interval]) * 1000) by (instance, pod)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]'
- - expr: max(greptime_cpu_limit_in_millicores{})
+ - expr: max(greptime_cpu_limit_in_millicores)
datasource:
type: prometheus
uid: ${metrics}
legendFormat: limit
+ - title: Flownode Memory per Instance
+ type: timeseries
+ description: Current memory usage by instance
+ unit: bytes
+ queries:
+ - expr: sum(process_resident_memory_bytes) by (instance, pod)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{ pod }}]'
+ - expr: max(greptime_memory_limit_in_bytes)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: limit
+ - title: Queries
+ panels:
+ - title: Query Rate by Protocol
+ type: timeseries
+ description: Query API call rates by protocol, collected from frontends.
+ unit: reqps
+ queries:
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql
+ - expr: sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres
+ - expr: sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql
+ - expr: sum(rate(greptime_servers_http_sql_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: http-sql
+ - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-query
+ - title: Query Latency by Protocol
+ type: timeseries
+ description: p95, p99, and average query latency by main frontend protocol.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-p95
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-p99
+ - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-p95
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-p99
+ - expr: histogram_quantile(0.95, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql-p95
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_servers_http_promql_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql-p99
+ - expr: sum(rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-avg
+ - expr: sum(rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-avg
+ - expr: sum(rate(greptime_servers_http_promql_elapsed_sum[$__rate_interval])) / sum(rate(greptime_servers_http_promql_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: promql-avg
+ - expr: sum(rate(greptime_frontend_grpc_handle_query_elapsed_sum[$__rate_interval])) / sum(rate(greptime_frontend_grpc_handle_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: grpc-query-avg
+ - title: Query Stage Latency
+ type: timeseries
+ description: p95 and p99 latency by query stage. Use stage labels to identify planning, scan, or merge bottlenecks.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p95-{{stage}}
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_query_stage_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99-{{stage}}
+ - title: Merge Scan Fan-out and Errors
+ type: timeseries
+ description: Merge-scan region fan-out and errors. High fan-out can explain slow distributed table scans.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_query_merge_scan_regions)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: regions-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_query_merge_scan_errors_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: errors-[{{instance}}]-[{{pod}}]
+ - title: Pushdown Fallback Errors
+ type: timeseries
+ description: Failed query pushdown fallback attempts. Non-zero values can indicate optimization paths that increase scan work.
+ unit: eps
+ queries:
+ - expr: sum(rate(greptime_push_down_fallback_errors_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: pushdown-fallback-errors
+ - title: PromQL Series Count
+ type: timeseries
+ description: Series count touched by PromQL queries. Correlate this with PromQL latency to identify cardinality-driven slowness.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_promql_series_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Connections and Prepared Statements
+ type: timeseries
+ description: MySQL/PostgreSQL connection and prepared-statement counts. Spikes can indicate client storms or leaks.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (greptime_servers_mysql_connection_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-connections-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_servers_postgres_connection_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-connections-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_servers_mysql_prepared_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: mysql-prepared-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_servers_postgres_prepared_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: postgres-prepared-[{{instance}}]-[{{pod}}]
- title: Frontend Requests
panels:
- title: HTTP QPS per Instance
@@ -309,9 +958,9 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]'
- - title: HTTP P99 per Instance
+ - title: HTTP P99 and Avg per Instance
type: timeseries
- description: HTTP P99 per Instance.
+ description: HTTP P99 and average per Instance.
unit: s
queries:
- expr: histogram_quantile(0.99, sum by(instance, pod, le, path, method, code) (rate(greptime_servers_http_requests_elapsed_bucket{path!~"/health|/metrics"}[$__rate_interval])))
@@ -319,99 +968,114 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99'
+ - expr: sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_sum{path!~"/health|/metrics"}[$__rate_interval])) / sum by(instance, pod, path, method, code) (rate(greptime_servers_http_requests_elapsed_count{path!~"/health|/metrics"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-avg'
- title: gRPC QPS per Instance
type: timeseries
description: gRPC QPS per Instance.
unit: reqps
queries:
- - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count{}[$__rate_interval]))
+ - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]'
- - title: gRPC P99 per Instance
+ - title: gRPC P99 and Avg per Instance
type: timeseries
- description: gRPC P99 per Instance.
+ description: gRPC P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, path, code) (rate(greptime_servers_grpc_requests_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{method}}]-[{{code}}]-p99'
+ - expr: sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_sum[$__rate_interval])) / sum by(instance, pod, path, code) (rate(greptime_servers_grpc_requests_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{path}}]-[{{code}}]-avg'
- title: MySQL QPS per Instance
type: timeseries
description: MySQL QPS per Instance.
unit: reqps
queries:
- - expr: sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count{}[$__rate_interval]))
+ - expr: sum by(pod, instance)(rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: MySQL P99 per Instance
+ - title: MySQL P99 and Avg per Instance
type: timeseries
- description: MySQL P99 per Instance.
+ description: MySQL P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(pod, instance, le) (rate(greptime_servers_mysql_query_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{ instance }}]-[{{ pod }}]-p99'
+ - expr: sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_mysql_query_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
- title: PostgreSQL QPS per Instance
type: timeseries
description: PostgreSQL QPS per Instance.
unit: reqps
queries:
- - expr: sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count{}[$__rate_interval]))
+ - expr: sum by(pod, instance)(rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: PostgreSQL P99 per Instance
+ - title: PostgreSQL P99 and Avg per Instance
type: timeseries
- description: PostgreSQL P99 per Instance.
+ description: PostgreSQL P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(pod,instance,le) (rate(greptime_servers_postgres_query_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - title: Frontend to Datanode
- panels:
- - title: Ingest Rows per Instance
- type: timeseries
- description: Ingestion rate by row as in each frontend
- unit: rowsps
- queries:
- - expr: sum by(instance, pod)(rate(greptime_table_operator_ingest_rows{}[$__rate_interval]))
+ - expr: sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_sum[$__rate_interval])) / sum by(pod, instance) (rate(greptime_servers_postgres_query_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
+ - title: Frontend to Datanode
+ panels:
- title: Region Call QPS per Instance
type: timeseries
description: Region Call QPS per Instance.
unit: ops
queries:
- - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count{}[$__rate_interval]))
+ - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]'
- - title: Region Call P99 per Instance
+ - title: Region Call P99 and Avg per Instance
type: timeseries
- description: Region Call P99 per Instance.
+ description: Region Call P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, request_type) (rate(greptime_grpc_region_request_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]'
- - title: 'Frontend Handle Bulk Insert Elapsed Time '
+ - expr: sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_sum[$__rate_interval])) / sum by(instance, pod, request_type) (rate(greptime_grpc_region_request_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{request_type}}]-avg'
+ - title: Frontend Handle Bulk Insert Elapsed Time
type: timeseries
description: Per-stage time for frontend to handle bulk insert requests
unit: s
@@ -426,34 +1090,156 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-P95'
- - title: Mito Engine
+ - title: Datanode
+ panels:
+ - title: Region Request Failures and Failed Inserts
+ type: timeseries
+ description: Datanode region request failures and failed inserts by instance.
+ unit: eps
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_request_fail_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: request-fail-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_datanode_region_failed_insert_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: failed-insert-[{{instance}}]-[{{pod}}]
+ - title: Write Rejects and Stalls
+ type: timeseries
+ description: Mito write rejects, write stall events, and active write stalling by datanode.
+ unit: short
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_mito_write_reject_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: reject-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_mito_write_stall_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: stall-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (greptime_mito_write_stalling_count)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: active-stalling-[{{instance}}]-[{{pod}}]
+ - title: Flush and Compaction Failures
+ type: timeseries
+ description: Mito flush and compaction failure rates by datanode.
+ unit: eps
+ queries:
+ - expr: sum by (instance, pod) (rate(greptime_mito_flush_failure_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: flush-[{{instance}}]-[{{pod}}]
+ - expr: sum by (instance, pod) (rate(greptime_mito_compaction_failure_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: compaction-[{{instance}}]-[{{pod}}]
+ - title: Mito GC Health
+ type: timeseries
+ description: Mito garbage-collection runs, errors, deleted files, orphaned index files, and skipped unparsable files.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_mito_gc_runs_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: runs
+ - expr: sum(rate(greptime_mito_gc_errors_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: errors
+ - expr: sum(rate(greptime_mito_gc_files_deleted_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: files-deleted
+ - expr: sum(rate(greptime_mito_gc_orphaned_index_files[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: orphaned-index-files
+ - expr: sum(rate(greptime_mito_gc_skipped_unparsable_files[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: skipped-unparsable
+ - title: Mito GC Duration
+ type: timeseries
+ description: P99 and average Mito garbage-collection duration by stage.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le, stage) (rate(greptime_mito_gc_duration_seconds_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}-p99'
+ - expr: sum by (stage) (rate(greptime_mito_gc_duration_seconds_sum[$__rate_interval])) / sum by (stage) (rate(greptime_mito_gc_duration_seconds_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{stage}}-avg'
+ - title: Storage
panels:
- title: Request OPS per Instance
type: timeseries
description: Request QPS per Instance.
unit: ops
queries:
- - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count{}[$__rate_interval]))
+ - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
- - title: Request P99 per Instance
+ - title: Request P99 and Avg per Instance
type: timeseries
- description: Request P99 per Instance.
+ description: Request P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, type) (rate(greptime_mito_handle_request_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
+ - expr: sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_sum[$__rate_interval])) / sum by(instance, pod, type) (rate(greptime_mito_handle_request_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg'
+ - title: Request Wait P99 and Avg per Worker
+ type: timeseries
+ description: Time Mito requests spend waiting before region worker handling. Use this with request service latency to distinguish queueing from execution time.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p95'
+ - expr: histogram_quantile(0.99, sum by(instance, pod, worker, le) (rate(greptime_mito_request_wait_time_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-p99'
+ - expr: sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_sum[$__rate_interval])) / sum by(instance, pod, worker) (rate(greptime_mito_request_wait_time_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{worker}}]-avg'
- title: Write Buffer per Instance
type: timeseries
description: Write Buffer per Instance.
unit: decbytes
queries:
- - expr: greptime_mito_write_buffer_bytes{}
+ - expr: greptime_mito_write_buffer_bytes
datasource:
type: prometheus
uid: ${metrics}
@@ -463,26 +1249,7 @@ groups:
description: Ingestion size by row counts.
unit: rowsps
queries:
- - expr: sum by (instance, pod) (rate(greptime_mito_write_rows_total{}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Flush OPS per Instance
- type: timeseries
- description: Flush QPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total{}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]'
- - title: Write Stall per Instance
- type: timeseries
- description: Write Stall per Instance.
- queries:
- - expr: sum by(instance, pod) (greptime_mito_write_stall_total{})
+ - expr: sum by (instance, pod) (rate(greptime_mito_write_rows_total[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
@@ -492,151 +1259,51 @@ groups:
description: Read Stage OPS per Instance.
unit: ops
queries:
- - expr: sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{ stage="total"}[$__rate_interval]))
+ - expr: sum by(instance, pod) (rate(greptime_mito_read_stage_elapsed_count{stage="total"}[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Read Stage P99 per Instance
+ - title: Read Stage P99 and Avg per Instance
type: timeseries
- description: Read Stage P99 per Instance.
+ description: Read Stage P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_read_stage_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]'
- - title: Write Stage P99 per Instance
- type: timeseries
- description: Write Stage P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket{}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]'
- - title: Compaction OPS per Instance
- type: timeseries
- description: Compaction OPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count{}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{ instance }}]-[{{pod}}]'
- - title: Compaction Elapsed Time per Instance by Stage
- type: timeseries
- description: Compaction latency by stage
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket{}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99'
- - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum{}[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count{}[$__rate_interval]))
+ - expr: sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_read_stage_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
- - title: Compaction P99 per Instance
+ - title: Write Stage P99 and Avg per Instance
type: timeseries
- description: Compaction P99 per Instance.
+ description: Write Stage P99 and average per Instance.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le,stage) (rate(greptime_mito_compaction_total_elapsed_bucket{}[$__rate_interval])))
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_write_stage_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-compaction'
- - title: WAL write size
- type: timeseries
- description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.
- unit: bytes
- queries:
- - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]'
+ - expr: sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_sum[$__rate_interval])) / sum by(instance, pod, stage) (rate(greptime_mito_write_stage_elapsed_count[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95'
- - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99'
- - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-throughput'
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
- title: Cached Bytes per Instance
type: timeseries
description: Cached Bytes per Instance.
unit: decbytes
queries:
- - expr: greptime_mito_cache_bytes{}
+ - expr: greptime_mito_cache_bytes
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
- - title: Inflight Compaction
- type: timeseries
- description: Ongoing compaction task count
- unit: none
- queries:
- - expr: greptime_mito_inflight_compaction_count
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: WAL sync duration seconds
- type: timeseries
- description: Raft engine (local disk) log store sync latency, p99
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - title: Log Store op duration seconds
- type: timeseries
- description: Write-ahead log operations latency at p99
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99'
- - title: Inflight Flush
- type: timeseries
- description: Ongoing flush task count
- unit: none
- queries:
- - expr: greptime_mito_inflight_flush_count
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Compaction Input/Output Bytes
- type: timeseries
- description: Compaction oinput output bytes
- unit: bytes
- queries:
- - expr: sum by(instance, pod) (greptime_mito_compaction_input_bytes)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-input'
- - expr: sum by(instance, pod) (greptime_mito_compaction_output_bytes)
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-output'
- title: Region Worker Handle Bulk Insert Requests
type: timeseries
description: Per-stage elapsed time for region worker to handle bulk insert region requests.
@@ -654,7 +1321,7 @@ groups:
legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-AVG'
- title: Active Series and Field Builders Count
type: timeseries
- description: Compaction oinput output bytes
+ description: Active series and field-builder counts per memtable by instance.
unit: none
queries:
- expr: sum by(instance, pod) (greptime_mito_memtable_active_series_count)
@@ -686,166 +1353,287 @@ groups:
type: timeseries
description: The local cache miss of the datanode.
queries:
- - expr: sum by (instance,pod, type) (rate(greptime_mito_cache_miss{}[$__rate_interval]))
+ - expr: sum by (instance,pod, type) (rate(greptime_mito_cache_miss[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]'
- - title: OpenDAL
+ - title: Flush and Compaction
panels:
- - title: QPS per Instance
+ - title: Flush OPS per Instance
type: timeseries
- description: QPS per Instance.
+ description: Flush QPS per Instance.
unit: ops
queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{}[$__rate_interval]))
+ - expr: sum by(instance, pod, reason) (rate(greptime_mito_flush_requests_total[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Read QPS per Instance
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{reason}}]'
+ - title: Flush Elapsed Time
type: timeseries
- description: Read QPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"read|Reader::read"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Read P99 per Instance
- type: timeseries
- description: Read P99 per Instance.
+ description: Mito flush p95 and p99 elapsed time by datanode and flush type. Use this to identify slow flush jobs.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read|Reader::read"}[$__rate_interval])))
+ - expr: histogram_quantile(0.95, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Write QPS per Instance
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95'
+ - expr: histogram_quantile(0.99, sum by (instance, pod, le, type) (rate(greptime_mito_flush_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
+ - expr: sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_sum[$__rate_interval])) / sum by (instance, pod, type) (rate(greptime_mito_flush_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-avg'
+ - title: Flush Throughput
type: timeseries
- description: Write QPS per Instance.
- unit: ops
+ description: Mito flushed bytes and flushed file rates. Use this with flush elapsed time to distinguish slow jobs from large jobs.
+ unit: Bps
queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{ operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+ - expr: sum by (instance, pod) (rate(greptime_mito_flush_bytes_total[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Write P99 per Instance
- type: timeseries
- description: Write P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation =~ "Writer::write|Writer::close|write"}[$__rate_interval])))
+ legendFormat: '[{{instance}}]-[{{pod}}]-bytes'
+ - expr: sum by (instance, pod) (rate(greptime_mito_flush_file_total[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: List QPS per Instance
+ legendFormat: '[{{instance}}]-[{{pod}}]-files'
+ - title: Inflight Flush
type: timeseries
- description: List QPS per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{ operation="list"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
- - title: List P99 per Instance
- type: timeseries
- description: List P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{ operation="list"}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
- - title: Other Requests per Instance
- type: timeseries
- description: Other Requests per Instance.
- unit: ops
- queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|write|list|stat"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Other Request P99 per Instance
- type: timeseries
- description: Other Request P99 per Instance.
- unit: s
- queries:
- - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{ operation!~"read|write|list|Writer::write|Writer::close|Reader::read"}[$__rate_interval])))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: Opendal traffic
- type: timeseries
- description: Total traffic as in bytes by instance and operation
- unit: decbytes
- queries:
- - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum{}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
- - title: OpenDAL errors per Instance
- type: timeseries
- description: OpenDAL error counts per Instance.
- queries:
- - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{ error!="NotFound"}[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]'
- - title: Remote WAL
- panels:
- - title: Triggered region flush total
- type: timeseries
- description: Triggered region flush total
+ description: Ongoing flush task count
unit: none
queries:
- - expr: meta_triggered_region_flush_total
+ - expr: greptime_mito_inflight_flush_count
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{topic_name}}'
- - title: Triggered region checkpoint total
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Compaction OPS per Instance
type: timeseries
- description: Triggered region checkpoint total
+ description: Compaction OPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{ instance }}]-[{{pod}}]'
+ - title: Inflight Compaction
+ type: timeseries
+ description: Ongoing compaction task count
unit: none
queries:
- - expr: meta_triggered_region_checkpoint_total
+ - expr: greptime_mito_inflight_compaction_count
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{topic_name}}'
- - title: Topic estimated replay size
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Compaction P99 and Avg per Instance
type: timeseries
- description: Topic estimated max replay size
+ description: Compaction P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le) (rate(greptime_mito_compaction_total_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p99'
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_mito_compaction_total_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
+ - title: Compaction Elapsed Time per Instance by Stage
+ type: timeseries
+ description: Compaction latency by stage
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, stage) (rate(greptime_mito_compaction_stage_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-p99'
+ - expr: sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_sum[$__rate_interval]))/sum by(instance, pod, stage) (rate(greptime_mito_compaction_stage_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{stage}}]-avg'
+ - title: Compaction Input/Output Bytes
+ type: timeseries
+ description: Compaction input and output bytes by datanode. Use this to correlate compaction latency with rewritten data volume.
+ unit: Bps
+ queries:
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_input_bytes[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-input'
+ - expr: sum by(instance, pod) (rate(greptime_mito_compaction_output_bytes[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-output'
+ - title: Index
+ panels:
+ - title: Index Apply Elapsed Time
+ type: timeseries
+ description: Index apply p95 and p99 elapsed time by index type. Slow apply can increase read latency for indexed predicates.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-p95'
+ - expr: histogram_quantile(0.99, sum by (le, type) (rate(greptime_index_apply_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-p99'
+ - title: Index Create Elapsed Time
+ type: timeseries
+ description: Index create p95 and p99 elapsed time by stage and index type. Slow stages can explain flush or compaction delays.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.95, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-{{stage}}-p95'
+ - expr: histogram_quantile(0.99, sum by (le, stage, type) (rate(greptime_index_create_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-{{stage}}-p99'
+ - title: Index Create Rows and Bytes
+ type: timeseries
+ description: Rows and bytes produced by index creation by index type. Spikes here can explain storage write pressure.
+ unit: rowsps
+ queries:
+ - expr: sum by (type) (rate(greptime_index_create_rows_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-rows'
+ - expr: sum by (type) (rate(greptime_index_create_bytes_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-bytes'
+ - title: Index Memory Usage
+ type: timeseries
+ description: Memory used while applying and creating indexes. Growth here can explain memory pressure during indexed flush or compaction work.
unit: bytes
queries:
- - expr: meta_topic_estimated_replay_size
+ - expr: greptime_index_apply_memory_usage
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{topic_name}}'
- - title: Kafka logstore's bytes traffic
+ legendFormat: apply
+ - expr: sum by (type) (greptime_index_create_memory_usage)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: create-{{type}}
+ - title: Index IO Bytes
type: timeseries
- description: Kafka logstore's bytes traffic
- unit: bytes
+ description: Index read and write byte rates by operation and file type for puffin and intermediate files.
+ unit: Bps
queries:
- - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])
+ - expr: sum by (type, file_type) (rate(greptime_index_io_bytes_total[$__rate_interval]))
datasource:
type: prometheus
uid: ${metrics}
- legendFormat: '{{pod}}-{{logstore}}'
+ legendFormat: '{{type}}-{{file_type}}'
+ - title: Index IO Operations
+ type: timeseries
+ description: Index IO operation rates by operation and file type, including read, write, seek, and flush operations.
+ unit: ops
+ queries:
+ - expr: sum by (type, file_type) (rate(greptime_index_io_op_total[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{type}}-{{file_type}}'
+ - title: Index Cache
+ type: timeseries
+ description: Index-related cache hits, misses, and evictions from Mito caches.
+ unit: ops
+ queries:
+ - expr: sum by (type) (rate(greptime_mito_cache_hit{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: hit-{{type}}
+ - expr: sum by (type) (rate(greptime_mito_cache_miss{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: miss-{{type}}
+ - expr: sum by (type, cause) (rate(greptime_mito_cache_eviction{type=~"index.*|vector_index|index_result"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: evict-{{type}}-{{cause}}
- title: Metasrv
panels:
+ - title: Inactive and Lease-expired Regions
+ type: timeseries
+ description: Inactive regions and expired region leases. Non-zero values indicate metasrv or routing health issues.
+ unit: short
+ queries:
+ - expr: sum(greptime_meta_inactive_regions)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: inactive-regions
+ - expr: sum(greptime_lease_expired_region)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: lease-expired
+ - title: Heartbeat Health
+ type: timeseries
+ description: Metasrv heartbeat receive rate, heartbeat connections, and frontend/datanode heartbeat send/receive counters.
+ unit: short
+ queries:
+ - expr: sum(rate(greptime_meta_heartbeat_rate[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: meta-recv-rate
+ - expr: sum(greptime_meta_heartbeat_connection_num)
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: meta-connections
+ - expr: sum(rate(greptime_frontend_heartbeat_send_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: frontend-send
+ - expr: sum(rate(greptime_frontend_heartbeat_recv_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: frontend-recv
+ - expr: sum(rate(greptime_datanode_heartbeat_send_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-send
+ - expr: sum(rate(greptime_datanode_heartbeat_recv_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: datanode-recv
- title: Region migration datanode
type: status-history
description: Counter of region migration by source and destination
@@ -865,7 +1653,7 @@ groups:
description: Counter of region migration error
unit: none
queries:
- - expr: greptime_meta_region_migration_error
+ - expr: rate(greptime_meta_region_migration_error[$__rate_interval])
datasource:
type: prometheus
uid: ${metrics}
@@ -900,6 +1688,11 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}} {{op}} {{type}} {{result}} p90'
+ - expr: sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_sum[$__rate_interval])) / sum by(pod, op, type, result) (rate(greptime_meta_rds_pg_sql_execute_elapsed_ms_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{op}}]-[{{type}}]-[{{result}}]-avg'
- title: Handler Execution Latency
type: timeseries
description: |
@@ -914,20 +1707,25 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}} {{name}} p90'
+ - expr: sum by(pod, name) (rate(greptime_meta_handler_execute_sum[$__rate_interval])) / sum by(pod, name) (rate(greptime_meta_handler_execute_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{name}}]-avg'
- title: Heartbeat Packet Size
type: timeseries
description: |
Shows p90 heartbeat message sizes, helping track network usage and identify anomalies in heartbeat payload.
unit: bytes
queries:
- - expr: histogram_quantile(0.9, sum by(pod, le) (greptime_meta_heartbeat_stat_memory_size_bucket))
+ - expr: histogram_quantile(0.9, sum by(pod, le) (rate(greptime_meta_heartbeat_stat_memory_size_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}}'
- title: Meta Heartbeat Receive Rate
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: Rate of heartbeats received by metasrv from datanodes and frontends.
unit: s
queries:
- expr: rate(greptime_meta_heartbeat_rate[$__rate_interval])
@@ -937,17 +1735,22 @@ groups:
legendFormat: '{{pod}}'
- title: Meta KV Ops Latency
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: p99 and average latency of metasrv key-value store operations by op and target.
unit: s
queries:
- - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (greptime_meta_kv_request_elapsed_bucket))
+ - expr: histogram_quantile(0.99, sum by(pod, le, op, target) (rate(greptime_meta_kv_request_elapsed_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '{{pod}}-{{op}} p99'
+ - expr: sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_sum[$__rate_interval])) / sum by(pod, op, target) (rate(greptime_meta_kv_request_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{op}}]-[{{target}}]-avg'
- title: Rate of meta KV Ops
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: Rate of metasrv key-value store operations by op.
unit: none
queries:
- expr: rate(greptime_meta_kv_request_elapsed_count[$__rate_interval])
@@ -957,35 +1760,35 @@ groups:
legendFormat: '{{pod}}-{{op}} p99'
- title: DDL Latency
type: timeseries
- description: Gauge of load information of each datanode, collected via heartbeat between datanode and metasrv. This information is for metasrv to schedule workloads.
+ description: p90 latency of metasrv DDL procedures (create/alter/drop table, create view/flow) by step.
unit: s
queries:
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_tables_bucket))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_tables_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateLogicalTables-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_table))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_table_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateTable-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_view))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_view_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateView-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_create_flow))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_create_flow_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: CreateFlow-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_drop_table))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_drop_table_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: DropTable-{{step}} p90
- - expr: histogram_quantile(0.9, sum by(le, pod, step) (greptime_meta_procedure_alter_table))
+ - expr: histogram_quantile(0.9, sum by(le, pod, step) (rate(greptime_meta_procedure_alter_table_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
@@ -993,9 +1796,9 @@ groups:
- title: Reconciliation stats
type: timeseries
description: Reconciliation stats
- unit: s
+ unit: ops
queries:
- - expr: greptime_meta_reconciliation_stats
+ - expr: rate(greptime_meta_reconciliation_stats[$__rate_interval])
datasource:
type: prometheus
uid: ${metrics}
@@ -1005,154 +1808,11 @@ groups:
description: Elapsed of Reconciliation steps
unit: s
queries:
- - expr: histogram_quantile(0.9, greptime_meta_reconciliation_procedure_bucket)
+ - expr: histogram_quantile(0.9, sum by(le, procedure_name, step) (rate(greptime_meta_reconciliation_procedure_bucket[$__rate_interval])))
datasource:
type: prometheus
uid: ${metrics}
legendFormat: '{{procedure_name}}-{{step}}-P90'
- - title: Flownode
- panels:
- - title: Flow Ingest / Output Rate
- type: timeseries
- description: Flow Ingest / Output Rate.
- queries:
- - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]'
- - title: Flow Ingest Latency
- type: timeseries
- description: Flow Ingest Latency.
- queries:
- - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p95'
- - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - title: Flow Operation Latency
- type: timeseries
- description: Flow Operation Latency.
- queries:
- - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95'
- - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
- - title: Flow Buffer Size per Instance
- type: timeseries
- description: Flow Buffer Size per Instance.
- queries:
- - expr: greptime_flow_input_buf_size
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]'
- - title: Flow Processing Error per Instance
- type: timeseries
- description: Flow Processing Error per Instance.
- queries:
- - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]'
- - title: Trigger
- panels:
- - title: Trigger Count
- type: timeseries
- description: Total number of triggers currently defined.
- queries:
- - expr: greptime_trigger_count{}
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- - title: Trigger Eval Elapsed
- type: timeseries
- description: Elapsed time for trigger evaluation, including query execution and condition evaluation.
- unit: s
- queries:
- - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p99'
- - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-p75'
- - title: Trigger Eval Failure Rate
- type: timeseries
- description: Rate of failed trigger evaluations.
- unit: none
- queries:
- - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval])
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- - title: Send Alert Elapsed
- type: timeseries
- description: Elapsed time to send trigger alerts to notification channels.
- unit: s
- queries:
- - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p99'
- - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{channel_type}}]-p75'
- - title: Send Alert Failure Rate
- type: timeseries
- description: Rate of failures when sending trigger alerts.
- unit: none
- queries:
- - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval])
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- - title: Save Alert Elapsed
- type: timeseries
- description: Elapsed time to persist trigger alert records.
- unit: s
- queries:
- - expr: "histogram_quantile(0.99, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p99'
- - expr: "histogram_quantile(0.75, \n rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])\n)"
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: '[{{instance}}]-[{{pod}}]-[{{storage_type}}]-p75'
- - title: Save Alert Failure Rate
- type: timeseries
- description: Rate of failures when persisting trigger alert records.
- unit: none
- queries:
- - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])
- datasource:
- type: prometheus
- uid: ${metrics}
- legendFormat: __auto
- title: Hotspot
panels:
- title: Hotspot Regions
@@ -1310,3 +1970,379 @@ groups:
type: prometheus
uid: ${metrics}
legendFormat: '{{stage}}'
+ - title: Object Store
+ panels:
+ - title: QPS per Instance
+ type: timeseries
+ description: QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Read QPS per Instance
+ type: timeseries
+ description: Read QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read|Reader::read"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Read P99 and Avg per Instance
+ type: timeseries
+ description: Read P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation=~"read|Reader::read"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"read|Reader::read"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"read|Reader::read"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: Write QPS per Instance
+ type: timeseries
+ description: Write QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Write P99 and Avg per Instance
+ type: timeseries
+ description: Write P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation =~ "Writer::write|Writer::close|write"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation=~"write|Writer::write|Writer::close"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation=~"write|Writer::write|Writer::close"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: List QPS per Instance
+ type: timeseries
+ description: List QPS per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
+ - title: List P99 and Avg per Instance
+ type: timeseries
+ description: List P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme) (rate(opendal_operation_duration_seconds_bucket{operation="list"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation="list"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation="list"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: Other Requests per Instance
+ type: timeseries
+ description: Other Requests per Instance.
+ unit: ops
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: Other Request P99 and Avg per Instance
+ type: timeseries
+ description: Other Request P99 and average per Instance.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(instance, pod, le, scheme, operation) (rate(opendal_operation_duration_seconds_bucket{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_sum{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval])) / sum by(instance, pod, scheme, operation) (rate(opendal_operation_duration_seconds_count{operation!~"read|Reader::read|write|Writer::write|Writer::close|list|stat"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-avg'
+ - title: Opendal traffic
+ type: timeseries
+ description: Total traffic as in bytes by instance and operation
+ unit: decbytes
+ queries:
+ - expr: sum by(instance, pod, scheme, operation) (rate(opendal_operation_bytes_sum[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]'
+ - title: OpenDAL errors per Instance
+ type: timeseries
+ description: OpenDAL error counts per Instance.
+ queries:
+ - expr: sum by(instance, pod, scheme, operation, error) (rate(opendal_operation_errors_total{error!="NotFound"}[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{scheme}}]-[{{operation}}]-[{{error}}]'
+ - title: WAL
+ panels:
+ - title: WAL write size
+ type: timeseries
+ description: Write-ahead logs write size as bytes. This chart includes stats of p95 and p99 size by instance, total WAL write rate.
+ unit: bytes
+ queries:
+ - expr: histogram_quantile(0.95, sum by(le,instance, pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p95'
+ - expr: histogram_quantile(0.99, sum by(le,instance,pod) (rate(raft_engine_write_size_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-req-size-p99'
+ - expr: sum by (instance, pod)(rate(raft_engine_write_size_sum[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-throughput'
+ - title: WAL sync duration seconds
+ type: timeseries
+ description: Raft engine (local disk) log store sync latency, p99
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(le, type, node, instance, pod) (rate(raft_engine_sync_log_duration_seconds_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p99'
+ - title: Log Store op duration seconds
+ type: timeseries
+ description: Write-ahead log operations latency at p99
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by(le,logstore,optype,instance, pod) (rate(greptime_logstore_op_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{logstore}}]-[{{optype}}]-p99'
+ - title: Triggered region flush total
+ type: timeseries
+ description: Triggered region flush total
+ unit: none
+ queries:
+ - expr: meta_triggered_region_flush_total
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{topic_name}}'
+ - title: Triggered region checkpoint total
+ type: timeseries
+ description: Triggered region checkpoint total
+ unit: none
+ queries:
+ - expr: meta_triggered_region_checkpoint_total
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{topic_name}}'
+ - title: Topic estimated replay size
+ type: timeseries
+ description: Topic estimated max replay size
+ unit: bytes
+ queries:
+ - expr: meta_topic_estimated_replay_size
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{topic_name}}'
+ - title: Kafka logstore's bytes traffic
+ type: timeseries
+ description: Kafka logstore's bytes traffic
+ unit: bytes
+ queries:
+ - expr: rate(greptime_logstore_kafka_client_bytes_total[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '{{pod}}-{{logstore}}'
+ - title: Flownode
+ panels:
+ - title: Flow Ingest / Output Rate
+ type: timeseries
+ description: Flow Ingest / Output Rate.
+ queries:
+ - expr: sum by(instance, pod, direction) (rate(greptime_flow_processed_rows[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{pod}}]-[{{instance}}]-[{{direction}}]'
+ - title: Flow Ingest Latency
+ type: timeseries
+ description: Flow Ingest Latency.
+ queries:
+ - expr: histogram_quantile(0.95, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p95'
+ - expr: histogram_quantile(0.99, sum(rate(greptime_flow_insert_elapsed_bucket[$__rate_interval])) by (le, instance, pod))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-p99'
+ - expr: sum by(instance, pod) (rate(greptime_flow_insert_elapsed_sum[$__rate_interval])) / sum by(instance, pod) (rate(greptime_flow_insert_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-avg'
+ - title: Flow Operation Latency
+ type: timeseries
+ description: Flow Operation Latency.
+ queries:
+ - expr: histogram_quantile(0.95, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p95'
+ - expr: histogram_quantile(0.99, sum(rate(greptime_flow_processing_time_bucket[$__rate_interval])) by (le,instance,pod,type))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{type}}]-p99'
+ - title: Flow Buffer Size per Instance
+ type: timeseries
+ description: Flow Buffer Size per Instance.
+ queries:
+ - expr: greptime_flow_input_buf_size
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]'
+ - title: Flow Processing Error per Instance
+ type: timeseries
+ description: Flow Processing Error per Instance.
+ queries:
+ - expr: sum by(instance,pod,code) (rate(greptime_flow_errors[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: '[{{instance}}]-[{{pod}}]-[{{code}}]'
+ - title: Trigger
+ panels:
+ - title: Trigger Count
+ type: timeseries
+ description: Total number of triggers currently defined.
+ queries:
+ - expr: greptime_trigger_count
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
+ - title: Trigger Eval Elapsed
+ type: timeseries
+ description: Elapsed time for trigger evaluation, including query execution and condition evaluation.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99
+ - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_evaluate_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p75
+ - expr: sum(rate(greptime_trigger_evaluate_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_evaluate_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: avg
+ - title: Trigger Eval Failure Rate
+ type: timeseries
+ description: Rate of failed trigger evaluations.
+ unit: none
+ queries:
+ - expr: rate(greptime_trigger_evaluate_failure_count[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
+ - title: Send Alert Elapsed
+ type: timeseries
+ description: Elapsed time to send trigger alerts to notification channels.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99
+ - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_send_alert_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p75
+ - expr: sum(rate(greptime_trigger_send_alert_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_send_alert_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: avg
+ - title: Send Alert Failure Rate
+ type: timeseries
+ description: Rate of failures when sending trigger alerts.
+ unit: none
+ queries:
+ - expr: rate(greptime_trigger_send_alert_failure_count[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
+ - title: Save Alert Elapsed
+ type: timeseries
+ description: Elapsed time to persist trigger alert records.
+ unit: s
+ queries:
+ - expr: histogram_quantile(0.99, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p99
+ - expr: histogram_quantile(0.75, sum by (le) (rate(greptime_trigger_save_alert_record_elapsed_bucket[$__rate_interval])))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: p75
+ - expr: sum(rate(greptime_trigger_save_alert_record_elapsed_sum[$__rate_interval])) / sum(rate(greptime_trigger_save_alert_record_elapsed_count[$__rate_interval]))
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: avg
+ - title: Save Alert Failure Rate
+ type: timeseries
+ description: Rate of failures when persisting trigger alert records.
+ unit: none
+ queries:
+ - expr: rate(greptime_trigger_save_alert_record_failure_count[$__rate_interval])
+ datasource:
+ type: prometheus
+ uid: ${metrics}
+ legendFormat: __auto
diff --git a/grafana/scripts/gen-dashboards.sh b/grafana/scripts/gen-dashboards.sh
index 9cf8cbd2a3..5087d5bde9 100755
--- a/grafana/scripts/gen-dashboards.sh
+++ b/grafana/scripts/gen-dashboards.sh
@@ -6,7 +6,11 @@ DAC_IMAGE=ghcr.io/zyy17/dac:20250423-522bd35
remove_instance_filters() {
# Remove the instance filters for the standalone dashboards.
- sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\",?//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" > "$STANDALONE_DASHBOARD_DIR/dashboard.json"
+ sed -E 's/instance=~\\"(\$datanode|\$frontend|\$metasrv|\$flownode)\\"[[:space:]]*,?[[:space:]]*//g' "$CLUSTER_DASHBOARD_DIR/dashboard.json" \
+ | sed -E 's/\{[[:space:]]*,[[:space:]]*/{/g' \
+ | sed -E 's/,[[:space:]]*\}/}/g' \
+ | sed -E 's/([A-Za-z_:][A-Za-z0-9_:]*)\{[[:space:]]*\}/\1/g' \
+ > "$STANDALONE_DASHBOARD_DIR/dashboard.json"
}
generate_intermediate_dashboards_and_docs() {
diff --git a/scripts/fetch-dashboard-assets.sh b/scripts/fetch-dashboard-assets.sh
index 006d5027a6..3ccebb9521 100755
--- a/scripts/fetch-dashboard-assets.sh
+++ b/scripts/fetch-dashboard-assets.sh
@@ -1,14 +1,17 @@
#!/usr/bin/env bash
# This script is used to download built dashboard assets from the "GreptimeTeam/dashboard" repository.
-set -ex
+set -e
declare -r SCRIPT_DIR=$(cd $(dirname ${0}) >/dev/null 2>&1 && pwd)
declare -r ROOT_DIR=$(dirname ${SCRIPT_DIR})
declare -r STATIC_DIR="$ROOT_DIR/src/servers/dashboard"
OUT_DIR="${1:-$SCRIPT_DIR}"
-RELEASE_VERSION="$(cat $STATIC_DIR/VERSION | tr -d '\t\r\n ')"
+DASHBOARD_REPOSITORY="${DASHBOARD_REPOSITORY:-GreptimeTeam/dashboard}"
+DASHBOARD_ASSET="${DASHBOARD_ASSET:-build.tar.gz}"
+DASHBOARD_GITHUB_TOKEN="${DASHBOARD_GITHUB_TOKEN:-${GH_TOKEN:-${GITHUB_TOKEN:-}}}"
+RELEASE_VERSION="${DASHBOARD_RELEASE_VERSION:-$(cat "$STATIC_DIR/VERSION" | tr -d '\t\r\n ')}"
echo "Downloading assets to dir: $OUT_DIR"
cd $OUT_DIR
@@ -22,8 +25,16 @@ fi
function retry_fetch() {
local url=$1
local filename=$2
+ local auth_args=()
- curl --connect-timeout 10 --retry 3 -fsSL $url --output $filename || {
+ if [[ -n "$DASHBOARD_GITHUB_TOKEN" ]]; then
+ auth_args=(
+ -H "Authorization: Bearer ${DASHBOARD_GITHUB_TOKEN}"
+ -H "Accept: application/octet-stream"
+ )
+ fi
+
+ curl --connect-timeout 10 --retry 3 -fsSL "${auth_args[@]}" "$url" --output "$filename" || {
echo "Failed to download $url"
echo "You may try to set http_proxy and https_proxy environment variables."
if [[ -z "$GITHUB_PROXY_URL" ]]; then
@@ -36,10 +47,10 @@ function retry_fetch() {
# Download the SHA256 checksum attached to the release. To verify the integrity
# of the download, this checksum will be used to check the download tar file
# containing the built dashboard assets.
-retry_fetch "${GITHUB_URL}/GreptimeTeam/dashboard/releases/download/${RELEASE_VERSION}/sha256.txt" sha256.txt
+retry_fetch "${GITHUB_URL}/${DASHBOARD_REPOSITORY}/releases/download/${RELEASE_VERSION}/sha256.txt" sha256.txt
# Download the tar file containing the built dashboard assets.
-retry_fetch "${GITHUB_URL}/GreptimeTeam/dashboard/releases/download/${RELEASE_VERSION}/build.tar.gz" build.tar.gz
+retry_fetch "${GITHUB_URL}/${DASHBOARD_REPOSITORY}/releases/download/${RELEASE_VERSION}/${DASHBOARD_ASSET}" "$DASHBOARD_ASSET"
# Verify the checksums match; exit if they don't.
case "$(uname -s)" in
@@ -55,8 +66,8 @@ case "$(uname -s)" in
esac
# Extract the assets and clean up.
-tar -xzf build.tar.gz -C "$STATIC_DIR"
+tar -xzf "$DASHBOARD_ASSET" -C "$STATIC_DIR"
rm sha256.txt
-rm build.tar.gz
+rm "$DASHBOARD_ASSET"
echo "Successfully download dashboard assets to $STATIC_DIR"
diff --git a/src/catalog/src/process_manager.rs b/src/catalog/src/process_manager.rs
index 49be14933c..8730821c3e 100644
--- a/src/catalog/src/process_manager.rs
+++ b/src/catalog/src/process_manager.rs
@@ -307,6 +307,7 @@ impl Debug for CancellableProcess {
pub struct SlowQueryTimer {
start: Instant,
stmt: QueryStatement,
+ schema_name: String,
threshold: Duration,
sample_ratio: f64,
record_type: SlowQueriesRecordType,
@@ -316,6 +317,7 @@ pub struct SlowQueryTimer {
impl SlowQueryTimer {
pub fn new(
stmt: QueryStatement,
+ schema_name: String,
threshold: Duration,
sample_ratio: f64,
record_type: SlowQueriesRecordType,
@@ -324,6 +326,7 @@ impl SlowQueryTimer {
Self {
start: Instant::now(),
stmt,
+ schema_name,
threshold,
sample_ratio,
record_type,
@@ -338,6 +341,7 @@ impl SlowQueryTimer {
cost: elapsed.as_millis() as u64,
threshold: self.threshold.as_millis() as u64,
query: "".to_string(),
+ schema_name: self.schema_name.clone(),
// The following fields are only used for PromQL queries.
is_promql: false,
@@ -388,6 +392,7 @@ impl SlowQueryTimer {
cost = slow_query_event.cost,
threshold = slow_query_event.threshold,
query = slow_query_event.query,
+ schema_name = slow_query_event.schema_name,
is_promql = slow_query_event.is_promql,
promql_range = slow_query_event.promql_range,
promql_step = slow_query_event.promql_step,
diff --git a/src/common/frontend/src/slow_query_event.rs b/src/common/frontend/src/slow_query_event.rs
index f9a2620283..d7ec630c5d 100644
--- a/src/common/frontend/src/slow_query_event.rs
+++ b/src/common/frontend/src/slow_query_event.rs
@@ -24,6 +24,7 @@ pub const SLOW_QUERY_TABLE_NAME: &str = "slow_queries";
pub const SLOW_QUERY_TABLE_COST_COLUMN_NAME: &str = "cost";
pub const SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME: &str = "threshold";
pub const SLOW_QUERY_TABLE_QUERY_COLUMN_NAME: &str = "query";
+pub const SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME: &str = "schema_name";
pub const SLOW_QUERY_TABLE_TIMESTAMP_COLUMN_NAME: &str = "timestamp";
pub const SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME: &str = "is_promql";
pub const SLOW_QUERY_TABLE_PROMQL_START_COLUMN_NAME: &str = "promql_start";
@@ -38,6 +39,7 @@ pub struct SlowQueryEvent {
pub cost: u64,
pub threshold: u64,
pub query: String,
+ pub schema_name: String,
pub is_promql: bool,
pub promql_range: Option,
pub promql_step: Option,
@@ -104,6 +106,12 @@ impl Event for SlowQueryEvent {
semantic_type: SemanticType::Field.into(),
..Default::default()
},
+ ColumnSchema {
+ column_name: SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME.to_string(),
+ datatype: ColumnDataType::String.into(),
+ semantic_type: SemanticType::Field.into(),
+ ..Default::default()
+ },
]
}
@@ -118,6 +126,7 @@ impl Event for SlowQueryEvent {
ValueData::U64Value(self.promql_step.unwrap_or(0)).into(),
ValueData::TimestampMillisecondValue(self.promql_start.unwrap_or(0)).into(),
ValueData::TimestampMillisecondValue(self.promql_end.unwrap_or(0)).into(),
+ ValueData::StringValue(self.schema_name.clone()).into(),
],
}])
}
@@ -126,3 +135,54 @@ impl Event for SlowQueryEvent {
self
}
}
+
+#[cfg(test)]
+mod tests {
+ use api::v1::value::ValueData;
+ use common_event_recorder::Event;
+
+ use super::*;
+
+ #[test]
+ fn slow_query_event_includes_schema() {
+ let event = SlowQueryEvent {
+ cost: 100,
+ threshold: 10,
+ query: "SELECT * FROM numbers".to_string(),
+ schema_name: "public".to_string(),
+ is_promql: false,
+ promql_range: None,
+ promql_step: None,
+ promql_start: None,
+ promql_end: None,
+ };
+
+ let schema = event.extra_schema();
+ let column_names = schema
+ .iter()
+ .map(|column| column.column_name.as_str())
+ .collect::>();
+ assert_eq!(
+ column_names,
+ vec![
+ SLOW_QUERY_TABLE_COST_COLUMN_NAME,
+ SLOW_QUERY_TABLE_THRESHOLD_COLUMN_NAME,
+ SLOW_QUERY_TABLE_QUERY_COLUMN_NAME,
+ SLOW_QUERY_TABLE_IS_PROMQL_COLUMN_NAME,
+ SLOW_QUERY_TABLE_PROMQL_RANGE_COLUMN_NAME,
+ SLOW_QUERY_TABLE_PROMQL_STEP_COLUMN_NAME,
+ SLOW_QUERY_TABLE_PROMQL_START_COLUMN_NAME,
+ SLOW_QUERY_TABLE_PROMQL_END_COLUMN_NAME,
+ SLOW_QUERY_TABLE_SCHEMA_NAME_COLUMN_NAME,
+ ]
+ );
+ assert_eq!(schema[8].semantic_type, SemanticType::Field as i32);
+
+ let rows = event.extra_rows().unwrap();
+ assert_eq!(rows.len(), 1);
+ assert_eq!(
+ rows[0].values[8].value_data,
+ Some(ValueData::StringValue("public".to_string()))
+ );
+ }
+}
diff --git a/src/common/function/src/admin/gc.rs b/src/common/function/src/admin/gc.rs
index ad2e3d3890..b5e067ce72 100644
--- a/src/common/function/src/admin/gc.rs
+++ b/src/common/function/src/admin/gc.rs
@@ -12,8 +12,6 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-use std::time::Duration;
-
use common_error::ext::BoxedError;
use common_macro::admin_fn;
use common_meta::rpc::procedure::{GcRegionsRequest, GcTableRequest};
@@ -30,7 +28,6 @@ use snafu::{ResultExt, ensure};
use crate::handlers::ProcedureServiceHandlerRef;
use crate::helper::cast_u64;
-const DEFAULT_GC_TIMEOUT: Duration = Duration::from_secs(60);
const DEFAULT_FULL_FILE_LISTING: bool = false;
#[admin_fn(
@@ -50,7 +47,7 @@ pub(crate) async fn gc_regions(
.gc_regions(GcRegionsRequest {
region_ids,
full_file_listing,
- timeout: DEFAULT_GC_TIMEOUT,
+ timeout: None,
})
.await?;
@@ -77,7 +74,7 @@ pub(crate) async fn gc_table(
schema_name,
table_name,
full_file_listing,
- timeout: DEFAULT_GC_TIMEOUT,
+ timeout: None,
})
.await?;
@@ -180,9 +177,18 @@ fn gc_table_signature() -> Signature {
#[cfg(test)]
mod tests {
+ use std::sync::{Arc, Mutex};
+
+ use api::v1::meta::ReconcileRequest;
+ use async_trait::async_trait;
+ use catalog::CatalogManagerRef;
+ use common_meta::rpc::procedure::{
+ GcResponse, ManageRegionFollowerRequest, MigrateRegionRequest, ProcedureStateResponse,
+ };
use session::context::QueryContext;
use super::*;
+ use crate::handlers::ProcedureServiceHandler;
#[test]
fn test_parse_gc_regions_params_with_full_file_listing() {
@@ -217,4 +223,80 @@ mod tests {
assert_eq!(table, "t");
assert!(full_file_listing);
}
+
+ #[tokio::test]
+ async fn test_gc_regions_uses_meta_gc_timeout_config() {
+ let handler = Arc::new(MockProcedureServiceHandler::default());
+ let handler_ref: ProcedureServiceHandlerRef = handler.clone();
+ let params = vec![ValueRef::UInt64(1), ValueRef::Boolean(true)];
+
+ super::gc_regions(&handler_ref, &QueryContext::arc(), ¶ms)
+ .await
+ .unwrap();
+
+ let request = handler.gc_regions_request.lock().unwrap().clone().unwrap();
+ assert_eq!(request.region_ids, vec![1]);
+ assert!(request.full_file_listing);
+ assert_eq!(request.timeout, None);
+ }
+
+ #[tokio::test]
+ async fn test_gc_table_uses_meta_gc_timeout_config() {
+ let handler = Arc::new(MockProcedureServiceHandler::default());
+ let handler_ref: ProcedureServiceHandlerRef = handler.clone();
+ let params = vec![ValueRef::String("public.t"), ValueRef::Boolean(true)];
+
+ super::gc_table(&handler_ref, &QueryContext::arc(), ¶ms)
+ .await
+ .unwrap();
+
+ let request = handler.gc_table_request.lock().unwrap().clone().unwrap();
+ assert_eq!(request.catalog_name, "greptime");
+ assert_eq!(request.schema_name, "public");
+ assert_eq!(request.table_name, "t");
+ assert!(request.full_file_listing);
+ assert_eq!(request.timeout, None);
+ }
+
+ #[derive(Default)]
+ struct MockProcedureServiceHandler {
+ gc_regions_request: Mutex>,
+ gc_table_request: Mutex>,
+ }
+
+ #[async_trait]
+ impl ProcedureServiceHandler for MockProcedureServiceHandler {
+ async fn migrate_region(&self, _request: MigrateRegionRequest) -> Result> {
+ unreachable!()
+ }
+
+ async fn reconcile(&self, _request: ReconcileRequest) -> Result> {
+ unreachable!()
+ }
+
+ async fn query_procedure_state(&self, _pid: &str) -> Result {
+ unreachable!()
+ }
+
+ async fn manage_region_follower(
+ &self,
+ _request: ManageRegionFollowerRequest,
+ ) -> Result<()> {
+ unreachable!()
+ }
+
+ fn catalog_manager(&self) -> &CatalogManagerRef {
+ unreachable!()
+ }
+
+ async fn gc_regions(&self, request: GcRegionsRequest) -> Result {
+ *self.gc_regions_request.lock().unwrap() = Some(request);
+ Ok(GcResponse::default())
+ }
+
+ async fn gc_table(&self, request: GcTableRequest) -> Result {
+ *self.gc_table_request.lock().unwrap() = Some(request);
+ Ok(GcResponse::default())
+ }
+ }
}
diff --git a/src/common/meta/src/ddl/comment_on.rs b/src/common/meta/src/ddl/comment_on.rs
index 65c2cbc0ec..c4970abb83 100644
--- a/src/common/meta/src/ddl/comment_on.rs
+++ b/src/common/meta/src/ddl/comment_on.rs
@@ -16,7 +16,7 @@ use std::sync::Arc;
use async_trait::async_trait;
use chrono::Utc;
-use common_catalog::format_full_table_name;
+use common_catalog::{format_full_flow_name, format_full_table_name};
use common_procedure::error::{FromJsonSnafu, Result as ProcedureResult, ToJsonSnafu};
use common_procedure::{Context as ProcedureContext, LockKey, Procedure, Status};
use common_telemetry::tracing::info;
@@ -38,8 +38,8 @@ use crate::key::flow::flow_info::{FlowInfoKey, FlowInfoValue};
use crate::key::table_info::{TableInfoKey, TableInfoValue};
use crate::key::table_name::TableNameKey;
use crate::key::{DeserializedValueWithBytes, FlowId, MetadataKey, MetadataValue};
-use crate::lock_key::{CatalogLock, FlowNameLock, SchemaLock, TableNameLock};
-use crate::rpc::ddl::{CommentObjectType, CommentOnTask};
+use crate::lock_key::{CatalogLock, FlowLock, FlowNameLock, SchemaLock, TableLock, TableNameLock};
+use crate::rpc::ddl::{CommentObjectId, CommentObjectType, CommentOnTask};
use crate::rpc::store::PutRequest;
pub struct CommentOnProcedure {
@@ -106,26 +106,29 @@ impl CommentOnProcedure {
}
async fn prepare_table_or_column(&mut self) -> Result<()> {
- let table_name_key = TableNameKey::new(
- &self.data.catalog_name,
- &self.data.schema_name,
- &self.data.object_name,
- );
+ let table_id = if let Some(table_id) = self.data.table_id {
+ table_id
+ } else {
+ let table_name_key = TableNameKey::new(
+ &self.data.catalog_name,
+ &self.data.schema_name,
+ &self.data.object_name,
+ );
- let table_id = self
- .context
- .table_metadata_manager
- .table_name_manager()
- .get(table_name_key)
- .await?
- .with_context(|| TableNotFoundSnafu {
- table_name: format_full_table_name(
- &self.data.catalog_name,
- &self.data.schema_name,
- &self.data.object_name,
- ),
- })?
- .table_id();
+ self.context
+ .table_metadata_manager
+ .table_name_manager()
+ .get(table_name_key)
+ .await?
+ .with_context(|| TableNotFoundSnafu {
+ table_name: format_full_table_name(
+ &self.data.catalog_name,
+ &self.data.schema_name,
+ &self.data.object_name,
+ ),
+ })?
+ .table_id()
+ };
let table_info = self
.context
@@ -198,17 +201,22 @@ impl CommentOnProcedure {
}
async fn prepare_flow(&mut self) -> Result<()> {
- let flow_name_value = self
- .context
- .flow_metadata_manager
- .flow_name_manager()
- .get(&self.data.catalog_name, &self.data.object_name)
- .await?
- .with_context(|| FlowNotFoundSnafu {
- flow_name: &self.data.object_name,
- })?;
-
- let flow_id = flow_name_value.flow_id();
+ let flow_id = if let Some(flow_id) = self.data.flow_id {
+ flow_id
+ } else {
+ self.context
+ .flow_metadata_manager
+ .flow_name_manager()
+ .get(&self.data.catalog_name, &self.data.object_name)
+ .await?
+ .with_context(|| FlowNotFoundSnafu {
+ flow_name: format_full_flow_name(
+ &self.data.catalog_name,
+ &self.data.object_name,
+ ),
+ })?
+ .flow_id()
+ };
let flow_info = self
.context
.flow_metadata_manager
@@ -216,7 +224,7 @@ impl CommentOnProcedure {
.get_raw(flow_id)
.await?
.with_context(|| FlowNotFoundSnafu {
- flow_name: &self.data.object_name,
+ flow_name: format_full_flow_name(&self.data.catalog_name, &self.data.object_name),
})?;
self.data.flow_id = Some(flow_id);
@@ -411,17 +419,23 @@ impl Procedure for CommentOnProcedure {
let lock_key = match self.data.object_type {
CommentObjectType::Table | CommentObjectType::Column => {
- vec![
+ let mut lock_key = vec![
CatalogLock::Read(catalog).into(),
SchemaLock::read(catalog, schema).into(),
- TableNameLock::new(catalog, schema, &self.data.object_name).into(),
- ]
+ ];
+ if let Some(table_id) = self.data.table_id {
+ lock_key.push(TableLock::Write(table_id).into());
+ }
+ lock_key.push(TableNameLock::new(catalog, schema, &self.data.object_name).into());
+ lock_key
}
CommentObjectType::Flow => {
- vec![
- CatalogLock::Read(catalog).into(),
- FlowNameLock::new(catalog, &self.data.object_name).into(),
- ]
+ let mut lock_key = vec![CatalogLock::Read(catalog).into()];
+ if let Some(flow_id) = self.data.flow_id {
+ lock_key.push(FlowLock::Write(flow_id).into());
+ }
+ lock_key.push(FlowNameLock::new(catalog, &self.data.object_name).into());
+ lock_key
}
};
@@ -466,6 +480,12 @@ pub struct CommentOnData {
impl CommentOnData {
pub fn new(task: CommentOnTask) -> Self {
+ let (table_id, flow_id) = match task.object_id {
+ Some(CommentObjectId::Table(table_id)) => (Some(table_id), None),
+ Some(CommentObjectId::Flow(flow_id)) => (None, Some(flow_id)),
+ None => (None, None),
+ };
+
Self {
state: CommentOnState::Prepare,
catalog_name: task.catalog_name,
@@ -474,9 +494,9 @@ impl CommentOnData {
object_name: task.object_name,
column_name: task.column_name,
comment: task.comment,
- table_id: None,
+ table_id,
table_info: None,
- flow_id: None,
+ flow_id,
flow_info: None,
is_unchanged: false,
}
diff --git a/src/common/meta/src/ddl_manager.rs b/src/common/meta/src/ddl_manager.rs
index a6192f06ba..a92c9b679a 100644
--- a/src/common/meta/src/ddl_manager.rs
+++ b/src/common/meta/src/ddl_manager.rs
@@ -574,9 +574,15 @@ impl DdlManager {
#[tracing::instrument(skip_all)]
pub async fn submit_comment_on_task(
&self,
- comment_on_task: CommentOnTask,
+ mut comment_on_task: CommentOnTask,
) -> Result<(ProcedureId, Option