fix(grafana): correct autopilot counters and show baseline reasons (#10160)

* fix(grafana): correct autopilot counters and show baseline reasons

Signed-off-by: WenyXu <wenymedia@gmail.com>

* fix(grafana): retain baseline totals when details are unavailable

Signed-off-by: WenyXu <wenymedia@gmail.com>

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>

grafana/dashboards/metrics/cluster/dashboard.json

@@ -13962,7 +13962,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (result) (\n  changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
+              "expr": "sum by (result) (\n  increase(greptime_region_balancer_actions_total[$__rate_interval])\n)",
               "hide": false,
               "instant": false,
               "legendFormat": "{{result}}",
@@ -14064,7 +14064,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
+              "expr": "sum by (gate, reason) (increase(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "{{gate}} / {{reason}}",
@@ -14399,7 +14399,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
+              "expr": "sum by (result) (increase(greptime_auto_repartition_actions_total[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "{{result}}",
@@ -14415,7 +14415,7 @@
             "type": "prometheus",
             "uid": "${metrics}"
           },
-          "description": "Auto repartition gate stop count by gate and reason",
+          "description": "Repartition gate stop increments. Baseline failures show detailed reasons when available; otherwise the aggregate invalid_baseline series is retained. Fallback is matched per source before aggregation to avoid double counting across mixed versions.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -14501,12 +14501,25 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
+              "expr": "sum by (gate, reason) (\n  increase(greptime_auto_repartition_gate_stop_total{reason!=\"invalid_baseline\"}[$__rate_interval])\n  or\n  (increase(greptime_auto_repartition_gate_stop_total{reason=\"invalid_baseline\"}[$__rate_interval])\n    unless ignoring (gate, reason)\n    increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))\n)",
               "hide": false,
               "instant": false,
               "legendFormat": "{{gate}} / {{reason}}",
               "range": true,
               "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${metrics}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (reason) (increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "baseline / {{reason}}",
+              "range": true,
+              "refId": "B"
             }
           ],
           "title": "Auto Repartition Gate Stops",

grafana/dashboards/metrics/cluster/dashboard.md

@@ -241,14 +241,20 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
 | Region Balancer Actions | `sum by (result) (
-  changes(greptime_region_balancer_actions_total[$__rate_interval])
+  increase(greptime_region_balancer_actions_total[$__rate_interval])
 )` | `timeseries` | Region balancer action count | `prometheus` | `short` | `{{result}}` |
-| Region Balancer Gate Stops | `sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))` | `timeseries` | Region balancer gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Region Balancer Gate Stops | `sum by (gate, reason) (increase(greptime_region_balancer_gate_stop_total[$__rate_interval]))` | `timeseries` | Region balancer gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
 | Region Balancer Datanodes | `sum by (state) (greptime_region_balancer_datanodes)` | `stat` | Region balancer datanode count by state | `prometheus` | `short` | `{{state}}` |
 | Region Balancer Regions | `sum by (state) (greptime_region_balancer_regions)` | `stat` | Region balancer region count by state | `prometheus` | `short` | `{{state}}` |
 | Region Balancer Datanode Stability | `sum by (state) (greptime_region_balancer_datanode_stability)` | `stat` | Region balancer datanode stability statistics by state | `prometheus` | `binBps` | `{{state}}` |
-| Auto Repartition Actions | `sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))` | `timeseries` | Auto repartition action count by result | `prometheus` | `short` | `{{result}}` |
-| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Auto Repartition Actions | `sum by (result) (increase(greptime_auto_repartition_actions_total[$__rate_interval]))` | `timeseries` | Auto repartition action count by result | `prometheus` | `short` | `{{result}}` |
+| Auto Repartition Gate Stops | `sum by (gate, reason) (
+  increase(greptime_auto_repartition_gate_stop_total{reason!="invalid_baseline"}[$__rate_interval])
+  or
+  (increase(greptime_auto_repartition_gate_stop_total{reason="invalid_baseline"}[$__rate_interval])
+    unless ignoring (gate, reason)
+    increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))
+)`<br/>`sum by (reason) (increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))` | `timeseries` | Repartition gate stop increments. Baseline failures show detailed reasons when available; otherwise the aggregate invalid_baseline series is retained. Fallback is matched per source before aggregation to avoid double counting across mixed versions. | `prometheus` | `short` | `{{gate}} / {{reason}}` |
 | Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
 | Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
 # Object Store

grafana/dashboards/metrics/cluster/dashboard.yaml

@@ -1904,7 +1904,7 @@ groups:
           queries:
             - expr: |-
                 sum by (result) (
-                  changes(greptime_region_balancer_actions_total[$__rate_interval])
+                  increase(greptime_region_balancer_actions_total[$__rate_interval])
                 )
               datasource:
                 type: prometheus
@@ -1915,7 +1915,7 @@ groups:
           description: Region balancer gate stop count by gate and reason
           unit: short
           queries:
-            - expr: sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))
+            - expr: sum by (gate, reason) (increase(greptime_region_balancer_gate_stop_total[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -1955,21 +1955,33 @@ groups:
           description: Auto repartition action count by result
           unit: short
           queries:
-            - expr: sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))
+            - expr: sum by (result) (increase(greptime_auto_repartition_actions_total[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '{{result}}'
         - title: Auto Repartition Gate Stops
           type: timeseries
-          description: Auto repartition gate stop count by gate and reason
+          description: Repartition gate stop increments. Baseline failures show detailed reasons when available; otherwise the aggregate invalid_baseline series is retained. Fallback is matched per source before aggregation to avoid double counting across mixed versions.
           unit: short
           queries:
-            - expr: sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))
+            - expr: |-
+                sum by (gate, reason) (
+                  increase(greptime_auto_repartition_gate_stop_total{reason!="invalid_baseline"}[$__rate_interval])
+                  or
+                  (increase(greptime_auto_repartition_gate_stop_total{reason="invalid_baseline"}[$__rate_interval])
+                    unless ignoring (gate, reason)
+                    increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))
+                )
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '{{gate}} / {{reason}}'
+            - expr: sum by (reason) (increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))
+              datasource:
+                type: prometheus
+                uid: ${metrics}
+              legendFormat: baseline / {{reason}}
         - title: Auto Repartition Sampling P99
           type: timeseries
           description: Auto repartition sampling elapsed time by stage

grafana/dashboards/metrics/standalone/dashboard.json

@@ -13962,7 +13962,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (result) (\n  changes(greptime_region_balancer_actions_total[$__rate_interval])\n)",
+              "expr": "sum by (result) (\n  increase(greptime_region_balancer_actions_total[$__rate_interval])\n)",
               "hide": false,
               "instant": false,
               "legendFormat": "{{result}}",
@@ -14064,7 +14064,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
+              "expr": "sum by (gate, reason) (increase(greptime_region_balancer_gate_stop_total[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "{{gate}} / {{reason}}",
@@ -14399,7 +14399,7 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))",
+              "expr": "sum by (result) (increase(greptime_auto_repartition_actions_total[$__rate_interval]))",
               "hide": false,
               "instant": false,
               "legendFormat": "{{result}}",
@@ -14415,7 +14415,7 @@
             "type": "prometheus",
             "uid": "${metrics}"
           },
-          "description": "Auto repartition gate stop count by gate and reason",
+          "description": "Repartition gate stop increments. Baseline failures show detailed reasons when available; otherwise the aggregate invalid_baseline series is retained. Fallback is matched per source before aggregation to avoid double counting across mixed versions.",
           "fieldConfig": {
             "defaults": {
               "color": {
@@ -14501,12 +14501,25 @@
                 "uid": "${metrics}"
               },
               "editorMode": "code",
-              "expr": "sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))",
+              "expr": "sum by (gate, reason) (\n  increase(greptime_auto_repartition_gate_stop_total{reason!=\"invalid_baseline\"}[$__rate_interval])\n  or\n  (increase(greptime_auto_repartition_gate_stop_total{reason=\"invalid_baseline\"}[$__rate_interval])\n    unless ignoring (gate, reason)\n    increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))\n)",
               "hide": false,
               "instant": false,
               "legendFormat": "{{gate}} / {{reason}}",
               "range": true,
               "refId": "A"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${metrics}"
+              },
+              "editorMode": "code",
+              "expr": "sum by (reason) (increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))",
+              "hide": false,
+              "instant": false,
+              "legendFormat": "baseline / {{reason}}",
+              "range": true,
+              "refId": "B"
             }
           ],
           "title": "Auto Repartition Gate Stops",

grafana/dashboards/metrics/standalone/dashboard.md

@@ -241,14 +241,20 @@ ORDER BY data_size DESC;` | `piechart` | Distribution of leader regions and data
 | Title | Query | Type | Description | Datasource | Unit | Legend Format |
 | --- | --- | --- | --- | --- | --- | --- |
 | Region Balancer Actions | `sum by (result) (
-  changes(greptime_region_balancer_actions_total[$__rate_interval])
+  increase(greptime_region_balancer_actions_total[$__rate_interval])
 )` | `timeseries` | Region balancer action count | `prometheus` | `short` | `{{result}}` |
-| Region Balancer Gate Stops | `sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))` | `timeseries` | Region balancer gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Region Balancer Gate Stops | `sum by (gate, reason) (increase(greptime_region_balancer_gate_stop_total[$__rate_interval]))` | `timeseries` | Region balancer gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
 | Region Balancer Datanodes | `sum by (state) (greptime_region_balancer_datanodes)` | `stat` | Region balancer datanode count by state | `prometheus` | `short` | `{{state}}` |
 | Region Balancer Regions | `sum by (state) (greptime_region_balancer_regions)` | `stat` | Region balancer region count by state | `prometheus` | `short` | `{{state}}` |
 | Region Balancer Datanode Stability | `sum by (state) (greptime_region_balancer_datanode_stability)` | `stat` | Region balancer datanode stability statistics by state | `prometheus` | `binBps` | `{{state}}` |
-| Auto Repartition Actions | `sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))` | `timeseries` | Auto repartition action count by result | `prometheus` | `short` | `{{result}}` |
-| Auto Repartition Gate Stops | `sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))` | `timeseries` | Auto repartition gate stop count by gate and reason | `prometheus` | `short` | `{{gate}} / {{reason}}` |
+| Auto Repartition Actions | `sum by (result) (increase(greptime_auto_repartition_actions_total[$__rate_interval]))` | `timeseries` | Auto repartition action count by result | `prometheus` | `short` | `{{result}}` |
+| Auto Repartition Gate Stops | `sum by (gate, reason) (
+  increase(greptime_auto_repartition_gate_stop_total{reason!="invalid_baseline"}[$__rate_interval])
+  or
+  (increase(greptime_auto_repartition_gate_stop_total{reason="invalid_baseline"}[$__rate_interval])
+    unless ignoring (gate, reason)
+    increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))
+)`<br/>`sum by (reason) (increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))` | `timeseries` | Repartition gate stop increments. Baseline failures show detailed reasons when available; otherwise the aggregate invalid_baseline series is retained. Fallback is matched per source before aggregation to avoid double counting across mixed versions. | `prometheus` | `short` | `{{gate}} / {{reason}}` |
 | Auto Repartition Sampling P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_sampling_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition sampling elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
 | Auto Repartition Executor P99 | `histogram_quantile(0.99, sum by (le, stage) (rate(greptime_auto_repartition_executor_elapsed_bucket[$__rate_interval])))` | `timeseries` | Auto repartition executor elapsed time by stage | `prometheus` | `s` | `{{stage}}` |
 # Object Store

grafana/dashboards/metrics/standalone/dashboard.yaml

@@ -1904,7 +1904,7 @@ groups:
           queries:
             - expr: |-
                 sum by (result) (
-                  changes(greptime_region_balancer_actions_total[$__rate_interval])
+                  increase(greptime_region_balancer_actions_total[$__rate_interval])
                 )
               datasource:
                 type: prometheus
@@ -1915,7 +1915,7 @@ groups:
           description: Region balancer gate stop count by gate and reason
           unit: short
           queries:
-            - expr: sum by (gate, reason) (changes(greptime_region_balancer_gate_stop_total[$__rate_interval]))
+            - expr: sum by (gate, reason) (increase(greptime_region_balancer_gate_stop_total[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
@@ -1955,21 +1955,33 @@ groups:
           description: Auto repartition action count by result
           unit: short
           queries:
-            - expr: sum by (result) (changes(greptime_auto_repartition_actions_total[$__rate_interval]))
+            - expr: sum by (result) (increase(greptime_auto_repartition_actions_total[$__rate_interval]))
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '{{result}}'
         - title: Auto Repartition Gate Stops
           type: timeseries
-          description: Auto repartition gate stop count by gate and reason
+          description: Repartition gate stop increments. Baseline failures show detailed reasons when available; otherwise the aggregate invalid_baseline series is retained. Fallback is matched per source before aggregation to avoid double counting across mixed versions.
           unit: short
           queries:
-            - expr: sum by (gate, reason) (changes(greptime_auto_repartition_gate_stop_total[$__rate_interval]))
+            - expr: |-
+                sum by (gate, reason) (
+                  increase(greptime_auto_repartition_gate_stop_total{reason!="invalid_baseline"}[$__rate_interval])
+                  or
+                  (increase(greptime_auto_repartition_gate_stop_total{reason="invalid_baseline"}[$__rate_interval])
+                    unless ignoring (gate, reason)
+                    increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))
+                )
               datasource:
                 type: prometheus
                 uid: ${metrics}
               legendFormat: '{{gate}} / {{reason}}'
+            - expr: sum by (reason) (increase(greptime_auto_repartition_invalid_baseline_total[$__rate_interval]))
+              datasource:
+                type: prometheus
+                uid: ${metrics}
+              legendFormat: baseline / {{reason}}
         - title: Auto Repartition Sampling P99
           type: timeseries
           description: Auto repartition sampling elapsed time by stage